View Javadoc
1   package io.jawk.frontend;
2   
3   /*-
4    * ╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲
5    * Jawk
6    * ჻჻჻჻჻჻
7    * Copyright (C) 2006 - 2026 MetricsHub
8    * ჻჻჻჻჻჻
9    * This program is free software: you can redistribute it and/or modify
10   * it under the terms of the GNU Lesser General Public License as
11   * published by the Free Software Foundation, either version 3 of the
12   * License, or (at your option) any later version.
13   *
14   * This program is distributed in the hope that it will be useful,
15   * but WITHOUT ANY WARRANTY; without even the implied warranty of
16   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17   * GNU General Lesser Public License for more details.
18   *
19   * You should have received a copy of the GNU General Lesser Public
20   * License along with this program.  If not, see
21   * <http://www.gnu.org/licenses/lgpl-3.0.html>.
22   * ╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱╲╱
23   */
24  
25  import java.io.File;
26  import java.io.IOException;
27  import java.io.LineNumberReader;
28  import java.io.PrintStream;
29  import java.nio.charset.StandardCharsets;
30  import java.nio.file.Files;
31  import java.nio.file.InvalidPathException;
32  import java.nio.file.Path;
33  import java.nio.file.Paths;
34  import java.util.ArrayDeque;
35  import java.util.ArrayList;
36  import java.util.Collections;
37  import java.util.Deque;
38  import java.util.EnumSet;
39  import java.util.HashMap;
40  import java.util.HashSet;
41  import java.util.List;
42  import java.util.Map;
43  import java.util.Set;
44  import java.util.function.Supplier;
45  import io.jawk.AwkSandboxException;
46  import io.jawk.NotImplementedError;
47  import io.jawk.backend.AVM;
48  import io.jawk.ext.ExtensionFunction;
49  import io.jawk.intermediate.Address;
50  import io.jawk.intermediate.AwkTuples;
51  import io.jawk.intermediate.BuiltinFunction;
52  import io.jawk.jrt.JRT;
53  import io.jawk.intermediate.Tuple;
54  import io.jawk.util.ScriptFileSource;
55  import io.jawk.util.ScriptSource;
56  import io.jawk.frontend.ast.LexerException;
57  import io.jawk.frontend.ast.ParserException;
58  
59  /**
60   * Converts the AWK script into a syntax tree,
61   * which is useful the backend that either compiles or interprets the script.
62   * <p>
63   * It contains the internal state of the parser and the lexer.
64   *
65   * @author Danny Daglas
66   */
67  public class AwkParser {
68  
69  	/**
70  	 * Flags that describe special behaviours of AST nodes. These replace the
71  	 * previous marker interfaces such as {@code Breakable} and
72  	 * {@code NonStatementAst}.
73  	 */
74  	private enum AstFlag {
75  		BREAKABLE,
76  		NEXTABLE,
77  		CONTINUEABLE,
78  		RETURNABLE,
79  		NON_STATEMENT
80  	}
81  
82  	/** Lexer token values. */
83  	enum Token {
84  		EOF,
85  		NEWLINE,
86  		SEMICOLON,
87  		ID,
88  		FUNC_ID,
89  		INTEGER,
90  		DOUBLE,
91  		STRING,
92  
93  		EQUALS,
94  
95  		AND,
96  		OR,
97  
98  		EQ,
99  		GT,
100 		GE,
101 		LT,
102 		LE,
103 		NE,
104 		NOT,
105 		PIPE,
106 		QUESTION_MARK,
107 		COLON,
108 		APPEND,
109 
110 		PLUS,
111 		MINUS,
112 		MULT,
113 		DIVIDE,
114 		MOD,
115 		POW,
116 		COMMA,
117 		MATCHES,
118 		NOT_MATCHES,
119 		DOLLAR,
120 
121 		INC,
122 		DEC,
123 
124 		PLUS_EQ,
125 		MINUS_EQ,
126 		MULT_EQ,
127 		DIV_EQ,
128 		MOD_EQ,
129 		POW_EQ,
130 
131 		OPEN_PAREN,
132 		CLOSE_PAREN,
133 		OPEN_BRACE,
134 		CLOSE_BRACE,
135 		OPEN_BRACKET,
136 		CLOSE_BRACKET,
137 
138 		BUILTIN_FUNC_NAME,
139 
140 		EXTENSION,
141 		TYPED_REGEXP,
142 		INDIRECT,
143 		DIRECTIVE_INCLUDE,
144 		DIRECTIVE_NAMESPACE,
145 		DIRECTIVE_UNSUPPORTED,
146 
147 		KW_FUNCTION,
148 		KW_BEGIN,
149 		KW_END,
150 		KW_BEGINFILE,
151 		KW_ENDFILE,
152 		KW_IN,
153 		KW_IF,
154 		KW_ELSE,
155 		KW_WHILE,
156 		KW_FOR,
157 		KW_DO,
158 		KW_RETURN,
159 		KW_EXIT,
160 		KW_NEXT,
161 		KW_NEXTFILE,
162 		KW_CONTINUE,
163 		KW_DELETE,
164 		KW_BREAK,
165 		KW_PRINT,
166 		KW_PRINTF,
167 		KW_GETLINE
168 	}
169 
170 	/**
171 	 * Contains a mapping of Jawk keywords to their
172 	 * token values.
173 	 * They closely correspond to AWK keywords, but with
174 	 * a few added extensions.
175 	 * <p>
176 	 * Keys are the keywords themselves, and values are the
177 	 * token values (equivalent to yytok values in lex/yacc).
178 	 * <p>
179 	 * <strong>Note:</strong> whether built-in AWK function names
180 	 * and special AWK variable names are formally keywords or not,
181 	 * they are not stored in this map. They are separated
182 	 * into other maps.
183 	 */
184 	private static final Map<String, Token> KEYWORDS = new HashMap<String, Token>();
185 
186 	static {
187 		// special keywords
188 		KEYWORDS.put("function", Token.KW_FUNCTION);
189 		KEYWORDS.put("BEGIN", Token.KW_BEGIN);
190 		KEYWORDS.put("END", Token.KW_END);
191 		KEYWORDS.put("BEGINFILE", Token.KW_BEGINFILE);
192 		KEYWORDS.put("ENDFILE", Token.KW_ENDFILE);
193 		KEYWORDS.put("in", Token.KW_IN);
194 
195 		// statements
196 		KEYWORDS.put("if", Token.KW_IF);
197 		KEYWORDS.put("else", Token.KW_ELSE);
198 		KEYWORDS.put("while", Token.KW_WHILE);
199 		KEYWORDS.put("for", Token.KW_FOR);
200 		KEYWORDS.put("do", Token.KW_DO);
201 		KEYWORDS.put("return", Token.KW_RETURN);
202 		KEYWORDS.put("exit", Token.KW_EXIT);
203 		KEYWORDS.put("next", Token.KW_NEXT);
204 		KEYWORDS.put("nextfile", Token.KW_NEXTFILE);
205 		KEYWORDS.put("continue", Token.KW_CONTINUE);
206 		KEYWORDS.put("delete", Token.KW_DELETE);
207 		KEYWORDS.put("break", Token.KW_BREAK);
208 
209 		// special-form functions
210 		KEYWORDS.put("print", Token.KW_PRINT);
211 		KEYWORDS.put("printf", Token.KW_PRINTF);
212 		KEYWORDS.put("getline", Token.KW_GETLINE);
213 	}
214 
215 	private static final int SP_IDX = 257;
216 	/**
217 	 * Contains a mapping of Jawk special variables to their
218 	 * variable token values.
219 	 * As of this writing, they correspond exactly to
220 	 * standard AWK variables, no more, no less.
221 	 * <p>
222 	 * Keys are the variable names themselves, and values are the
223 	 * variable token values.
224 	 */
225 	private static final Map<String, Integer> SPECIAL_VAR_NAMES = new HashMap<String, Integer>();
226 
227 	static {
228 		SPECIAL_VAR_NAMES.put("NR", SP_IDX);
229 		SPECIAL_VAR_NAMES.put("FNR", SP_IDX);
230 		SPECIAL_VAR_NAMES.put("NF", SP_IDX);
231 		SPECIAL_VAR_NAMES.put("FS", SP_IDX);
232 		SPECIAL_VAR_NAMES.put("RS", SP_IDX);
233 		SPECIAL_VAR_NAMES.put("OFS", SP_IDX);
234 		SPECIAL_VAR_NAMES.put("ORS", SP_IDX);
235 		SPECIAL_VAR_NAMES.put("RSTART", SP_IDX);
236 		SPECIAL_VAR_NAMES.put("RLENGTH", SP_IDX);
237 		SPECIAL_VAR_NAMES.put("FILENAME", SP_IDX);
238 		SPECIAL_VAR_NAMES.put("SUBSEP", SP_IDX);
239 		SPECIAL_VAR_NAMES.put("CONVFMT", SP_IDX);
240 		SPECIAL_VAR_NAMES.put("OFMT", SP_IDX);
241 		SPECIAL_VAR_NAMES.put("ENVIRON", SP_IDX);
242 		SPECIAL_VAR_NAMES.put("ARGC", SP_IDX);
243 		SPECIAL_VAR_NAMES.put("ARGV", SP_IDX);
244 		SPECIAL_VAR_NAMES.put("IGNORECASE", SP_IDX);
245 		SPECIAL_VAR_NAMES.put("ERRNO", SP_IDX);
246 		SPECIAL_VAR_NAMES.put("ARGIND", SP_IDX);
247 	}
248 
249 	/**
250 	 * Defined as concrete implementation class (not an
251 	 * interface reference) as to not clutter the interface
252 	 * with methods appropriate for private access, only.
253 	 */
254 	private final AwkSymbolTableImpl symbolTable = new AwkSymbolTableImpl();
255 
256 	private final Map<String, ExtensionFunction> extensions;
257 
258 	/** POSIX compile-time mode: rejects gawk syntax such as arrays of arrays and typed regexps. */
259 	private final boolean posix;
260 
261 	/** Whether the compiling engine permits source inclusion from the filesystem. */
262 	private final boolean sourceIncludeAllowed;
263 
264 	/**
265 	 * <p>
266 	 * Constructor for AwkParser.
267 	 * </p>
268 	 *
269 	 * @param extensions a {@link java.util.Map} object
270 	 * @param posix {@code true} to enforce POSIX compile-time behavior
271 	 */
272 	public AwkParser(Map<String, ExtensionFunction> extensions, boolean posix) {
273 		this(extensions, posix, true);
274 	}
275 
276 	/**
277 	 * Creates a parser with an explicit source-inclusion policy.
278 	 *
279 	 * @param extensions extension functions available during parsing
280 	 * @param posix {@code true} to enforce POSIX compile-time behavior
281 	 * @param sourceIncludeAllowed {@code true} to permit {@code @include}
282 	 */
283 	public AwkParser(Map<String, ExtensionFunction> extensions, boolean posix, boolean sourceIncludeAllowed) {
284 		this.extensions = extensions == null ?
285 				Collections.emptyMap() :
286 				Collections.unmodifiableMap(new HashMap<>(extensions));
287 		this.posix = posix;
288 		this.sourceIncludeAllowed = sourceIncludeAllowed;
289 	}
290 
291 	/**
292 	 * Returns whether the keyword is disabled in the current compile-time
293 	 * mode. BEGINFILE and ENDFILE are gawk extensions: in POSIX mode they are
294 	 * not special and lex as plain identifiers, exactly like
295 	 * {@code gawk --posix}.
296 	 *
297 	 * @param keywordToken the keyword token to inspect
298 	 * @return {@code true} when the keyword must be treated as an identifier
299 	 */
300 	private boolean isDisabledKeyword(Token keywordToken) {
301 		return posix && (keywordToken == Token.KW_BEGINFILE || keywordToken == Token.KW_ENDFILE);
302 	}
303 
304 	private boolean isAwkNamespaceIdentifier(String identifier) {
305 		int separator = identifier.indexOf("::");
306 		if (separator >= 0) {
307 			return "awk".equals(identifier.substring(0, separator));
308 		}
309 		return "awk".equals(currentNamespace);
310 	}
311 
312 	private String awkNamespaceComponent(String identifier) {
313 		return identifier.startsWith("awk::") ? identifier.substring("awk::".length()) : identifier;
314 	}
315 
316 	private String qualifyGlobalIdentifier(String identifier) {
317 		int separator = identifier.indexOf("::");
318 		if (separator >= 0) {
319 			return "awk".equals(identifier.substring(0, separator)) ? identifier.substring(separator + 2) : identifier;
320 		}
321 		if ("awk".equals(currentNamespace) || isAllUppercaseIdentifier(identifier)) {
322 			return identifier;
323 		}
324 		return currentNamespace + "::" + identifier;
325 	}
326 
327 	private boolean isAllUppercaseIdentifier(String identifier) {
328 		if (identifier.isEmpty()) {
329 			return false;
330 		}
331 		for (int i = 0; i < identifier.length(); i++) {
332 			char ch = identifier.charAt(i);
333 			if (ch < 'A' || ch > 'Z') {
334 				return false;
335 			}
336 		}
337 		return true;
338 	}
339 
340 	private List<ScriptSource> scriptSources;
341 	private int scriptSourcesCurrentIndex;
342 	private ScriptSource currentScriptSource;
343 	private LineNumberReader reader;
344 	private int c;
345 	private Token token;
346 	private String pendingIndirectIdentifier;
347 	private boolean pendingColon;
348 	private String currentNamespace = "awk";
349 	private long conditionPairCount;
350 	private final Deque<SourceState> includedSourceStack = new ArrayDeque<SourceState>();
351 	private final Set<Path> includedSourcePaths = new HashSet<Path>();
352 	private final Set<Path> topLevelSourcePaths = new HashSet<Path>();
353 
354 	private StringBuffer text = new StringBuffer();
355 	private StringBuffer string = new StringBuffer();
356 	private StringBuffer regexp = new StringBuffer();
357 
358 	private static final class SourceState {
359 		private final ScriptSource scriptSource;
360 		private final LineNumberReader reader;
361 		private final int currentCharacter;
362 		private final String namespace;
363 
364 		private SourceState(ScriptSource scriptSourceParam, LineNumberReader readerParam, int currentCharacterParam,
365 				String namespaceParam) {
366 			scriptSource = scriptSourceParam;
367 			reader = readerParam;
368 			currentCharacter = currentCharacterParam;
369 			namespace = namespaceParam;
370 		}
371 	}
372 
373 	private void read() throws IOException {
374 		text.append((char) c);
375 		c = reader.read();
376 		// completely bypass \r's
377 		while (c == '\r') {
378 			c = reader.read();
379 		}
380 	}
381 
382 	/**
383 	 * Consumes the exponent part of a numeric constant ({@code e} or {@code E},
384 	 * an optional sign, and at least one digit) when the current character
385 	 * starts one. When the characters after the {@code e}/{@code E} do not form
386 	 * a valid exponent, nothing is consumed, so the {@code e}/{@code E} starts
387 	 * the next token: {@code 1e} is the number {@code 1} followed by the
388 	 * identifier {@code e}, as in gawk.
389 	 *
390 	 * @return whether an exponent part was consumed
391 	 * @throws IOException upon an I/O error while reading the script source
392 	 */
393 	private boolean readExponent() throws IOException {
394 		if (c != 'e' && c != 'E') {
395 			return false;
396 		}
397 		// Peek past the 'e'/'E' (and an optional sign) for a digit before
398 		// consuming anything: the lexer keeps only one character of lookahead
399 		// in c, so the decision must be made without advancing.
400 		reader.mark(3);
401 		int first = reader.read();
402 		boolean validExponent;
403 		if (first == '+' || first == '-') {
404 			validExponent = Character.isDigit(reader.read());
405 		} else {
406 			validExponent = Character.isDigit(first);
407 		}
408 		reader.reset();
409 		if (!validExponent) {
410 			return false;
411 		}
412 		read();
413 		if (c == '+' || c == '-') {
414 			read();
415 		}
416 		while (c > 0 && Character.isDigit(c)) {
417 			read();
418 		}
419 		return true;
420 	}
421 
422 	/**
423 	 * Advances to the next readable source at a token boundary when the current
424 	 * reader has reached end-of-file. Deferring this transition until the current
425 	 * token is complete preserves the source and namespace used to classify its
426 	 * final token. Included files are unwound first in LIFO order, restoring the
427 	 * including reader, its unread character, and its namespace. Once the include
428 	 * stack is empty, parsing continues with the next top-level script source,
429 	 * whose namespace starts at {@code awk}.
430 	 *
431 	 * @throws IOException if a source cannot be closed or read
432 	 */
433 	private void advancePastEndOfSource() throws IOException {
434 		while (c < 0) {
435 			if (!includedSourceStack.isEmpty()) {
436 				reader.close();
437 				SourceState previous = includedSourceStack.pop();
438 				currentScriptSource = previous.scriptSource;
439 				reader = previous.reader;
440 				c = previous.currentCharacter;
441 				currentNamespace = previous.namespace;
442 			} else if ((scriptSourcesCurrentIndex + 1) < scriptSources.size()) {
443 				scriptSourcesCurrentIndex++;
444 				currentScriptSource = scriptSources.get(scriptSourcesCurrentIndex);
445 				reader = new LineNumberReader(currentScriptSource.getReader());
446 				currentNamespace = "awk";
447 				c = reader.read();
448 				while (c == '\r') {
449 					c = reader.read();
450 				}
451 			} else {
452 				return;
453 			}
454 		}
455 	}
456 
457 	/**
458 	 * Skip all whitespaces and comments
459 	 *
460 	 * @throws IOException
461 	 */
462 	private void skipWhitespaces() throws IOException {
463 		while (c == ' ' || c == '\t' || c == '#' || c == '\n') {
464 			if (c == '#') {
465 				while (c >= 0 && c != '\n') {
466 					read();
467 				}
468 			}
469 			read();
470 		}
471 	}
472 
473 	/**
474 	 * Parse the script streamed by script_reader. Build and return the
475 	 * root of the abstract syntax tree which represents the Jawk script.
476 	 *
477 	 * @param localScriptSources List of script sources
478 	 * @return The abstract syntax tree of this script.
479 	 * @throws java.io.IOException upon an IO error.
480 	 */
481 	public AstNode parse(List<ScriptSource> localScriptSources) throws IOException {
482 		if (localScriptSources == null || localScriptSources.isEmpty()) {
483 			throw new IOException("No script sources supplied");
484 		}
485 		this.scriptSources = Collections.unmodifiableList(new ArrayList<>(localScriptSources));
486 		scriptSourcesCurrentIndex = 0;
487 		currentScriptSource = this.scriptSources.get(scriptSourcesCurrentIndex);
488 		reader = new LineNumberReader(currentScriptSource.getReader());
489 		currentNamespace = "awk";
490 		includedSourceStack.clear();
491 		resetIncludedSourcePaths();
492 		pendingIndirectIdentifier = null;
493 		pendingColon = false;
494 		read();
495 		lexer();
496 		return SCRIPT();
497 	}
498 
499 	/**
500 	 * Parse a single AWK expression and return the corresponding AST.
501 	 *
502 	 * @param expressionSource The expression to parse (not a statement or rule, just an expression)
503 	 * @return tuples representing the expression
504 	 * @throws IOException upon an IO error or parsing error
505 	 */
506 	public AstNode parseExpression(ScriptSource expressionSource) throws IOException {
507 
508 		// Sanity check
509 		if (expressionSource == null) {
510 			throw new IOException("No source supplied");
511 		}
512 
513 		// Reader of the expression
514 		this.scriptSources = Collections.singletonList(expressionSource);
515 		scriptSourcesCurrentIndex = 0;
516 		currentScriptSource = expressionSource;
517 		reader = new LineNumberReader(currentScriptSource.getReader());
518 		currentNamespace = "awk";
519 		includedSourceStack.clear();
520 		resetIncludedSourcePaths();
521 		pendingIndirectIdentifier = null;
522 		pendingColon = false;
523 
524 		// Initialize the lexer
525 		read();
526 		lexer();
527 
528 		// An expression is a TERNARY_EXPRESSION
529 		return EXPRESSION_TO_EVALUATE();
530 	}
531 
532 	private void resetIncludedSourcePaths() throws IOException {
533 		includedSourcePaths.clear();
534 		topLevelSourcePaths.clear();
535 		for (ScriptSource source : scriptSources) {
536 			if (source instanceof ScriptFileSource) {
537 				String filePath = ((ScriptFileSource) source).getFilePath();
538 				Path sourcePath = Paths.get(filePath).toRealPath();
539 				includedSourcePaths.add(sourcePath);
540 				topLevelSourcePaths.add(sourcePath);
541 			}
542 		}
543 	}
544 
545 	private LexerException lexerException(String msg) {
546 		return new LexerException(msg, currentScriptSource.getDescription(), reader.getLineNumber());
547 	}
548 
549 	/**
550 	 * Returns the current 1-based source line number to stamp onto AST nodes that
551 	 * will later emit tuple line markers for runtime error reporting.
552 	 *
553 	 * @return current source line number using 1-based counting
554 	 */
555 	private int currentSourceLineNumber() {
556 		return reader.getLineNumber() + 1;
557 	}
558 
559 	/**
560 	 * Reads the string and handle all escape codes.
561 	 *
562 	 * @throws IOException
563 	 */
564 	private void readString() throws IOException {
565 		string.setLength(0);
566 
567 		while (token != Token.EOF && c > 0 && c != '"' && c != '\n') {
568 			if (c == '\\') {
569 				read();
570 				switch (c) {
571 				case 'n':
572 					string.append('\n');
573 					break;
574 				case 't':
575 					string.append('\t');
576 					break;
577 				case 'r':
578 					string.append('\r');
579 					break;
580 				case 'a':
581 					string.append('\007');
582 					break; // BEL 0x07
583 				case 'b':
584 					string.append('\010');
585 					break; // BS 0x08
586 				case 'f':
587 					string.append('\014');
588 					break; // FF 0x0C
589 				case 'v':
590 					string.append('\013');
591 					break; // VT 0x0B
592 				// Octal notation: \N \NN \NNN
593 				case '0':
594 				case '1':
595 				case '2':
596 				case '3':
597 				case '4':
598 				case '5':
599 				case '6':
600 				case '7': {
601 					int octalChar = c - '0';
602 					read();
603 					if (c >= '0' && c <= '7') {
604 						octalChar = (octalChar << 3) + c - '0';
605 						read();
606 						if (c >= '0' && c <= '7') {
607 							octalChar = (octalChar << 3) + c - '0';
608 							read();
609 						}
610 					}
611 					string.append((char) octalChar);
612 					continue;
613 				}
614 				// Hexadecimal notation: \xN \xNN
615 				case 'x': {
616 					int hexChar = 0;
617 					read();
618 					if (c >= '0' && c <= '9') {
619 						hexChar = c - '0';
620 					} else if (c >= 'A' && c <= 'F') {
621 						hexChar = c - 'A' + 10;
622 					} else if (c >= 'a' && c <= 'f') {
623 						hexChar = c - 'a' + 10;
624 					} else {
625 						string.append('x');
626 						continue;
627 					}
628 					read();
629 					if (c >= '0' && c <= '9') {
630 						hexChar = (hexChar << 4) + c - '0';
631 					} else if (c >= 'A' && c <= 'F') {
632 						hexChar = (hexChar << 4) + c - 'A' + 10;
633 					} else if (c >= 'a' && c <= 'f') {
634 						hexChar = (hexChar << 4) + c - 'a' + 10;
635 					} else {
636 						// Append what we already have, and continue directly, because we already have read the next char
637 						string.append((char) hexChar);
638 						continue;
639 					}
640 					string.append((char) hexChar);
641 					break;
642 				}
643 				default:
644 					string.append((char) c);
645 					break; // Remove the backslash
646 				}
647 			} else {
648 				string.append((char) c);
649 			}
650 			read();
651 		}
652 		if (token == Token.EOF || c == '\n' || c <= 0) {
653 			throw lexerException("Unterminated string: " + text);
654 		}
655 		read();
656 	}
657 
658 	/**
659 	 * Reads the regular expression (between slashes '/') and handle '\/'.
660 	 * A slash within a bracket expression (e.g. {@code /[/]/}) does not
661 	 * terminate the regular expression, per POSIX ERE bracket semantics.
662 	 *
663 	 * @throws IOException
664 	 */
665 	private void readRegexp() throws IOException {
666 		regexp.setLength(0);
667 
668 		boolean inBracket = false;
669 		while (token != Token.EOF && c > 0 && (c != '/' || inBracket) && c != '\n') {
670 			if (c == '\\') {
671 				read();
672 				if (c != '/') {
673 					regexp.append('\\');
674 				}
675 				regexp.append((char) c);
676 				read();
677 				continue;
678 			}
679 			if (!inBracket && c == '[') {
680 				inBracket = true;
681 				regexp.append((char) c);
682 				read();
683 				// a ']' right after '[' (or after '[^') is a literal ']'
684 				if (c == '^') {
685 					regexp.append((char) c);
686 					read();
687 				}
688 				if (c == ']') {
689 					regexp.append((char) c);
690 					read();
691 				}
692 				continue;
693 			}
694 			if (inBracket && c == '[') {
695 				regexp.append((char) c);
696 				read();
697 				// POSIX character class, collating element, or equivalence
698 				// class ([:alpha:], [.x.], [=e=]): its closing ']' does not
699 				// end the outer bracket expression.
700 				if (c == ':' || c == '.' || c == '=') {
701 					int delimiter = c;
702 					boolean closed = false;
703 					while (token != Token.EOF && c > 0 && c != '\n' && !closed) {
704 						int previous = c;
705 						regexp.append((char) c);
706 						read();
707 						if (previous == delimiter && c == ']') {
708 							regexp.append((char) c);
709 							read();
710 							closed = true;
711 						}
712 					}
713 				}
714 				continue;
715 			}
716 			if (inBracket && c == ']') {
717 				inBracket = false;
718 			}
719 			regexp.append((char) c);
720 			read();
721 		}
722 		if (token == Token.EOF || c == '\n' || c <= 0) {
723 			throw lexerException("Unterminated string: " + text);
724 		}
725 		read();
726 	}
727 
728 	private Token lexer(Token expectedToken) throws IOException {
729 		if (token != expectedToken) {
730 			throw parserException("Expecting " + expectedToken.name() + ". Found: " + token.name() + " (" + text + ")");
731 		}
732 		return lexer();
733 	}
734 
735 	private Token lexer() throws IOException {
736 		// clear whitespace
737 		while (true) {
738 			advancePastEndOfSource();
739 			if (c < 0 || c != ' ' && c != '\t' && c != '#' && c != '\\') {
740 				break;
741 			}
742 			if (c == '\\') {
743 				read();
744 				if (c == '\n') {
745 					read();
746 				}
747 				continue;
748 			}
749 			if (c == '#') {
750 				// kill comment
751 				while (c >= 0 && c != '\n') {
752 					read();
753 				}
754 			} else {
755 				read();
756 			}
757 		}
758 		text.setLength(0);
759 		if (pendingColon) {
760 			pendingColon = false;
761 			token = Token.COLON;
762 			return token;
763 		}
764 		if (pendingIndirectIdentifier != null) {
765 			text.append(pendingIndirectIdentifier);
766 			pendingIndirectIdentifier = null;
767 			token = Token.ID;
768 			return token;
769 		}
770 		if (c < 0) {
771 			token = Token.EOF;
772 			return token;
773 		}
774 		if (c == ',') {
775 			read();
776 			skipWhitespaces();
777 			token = Token.COMMA;
778 			return token;
779 		}
780 		if (c == '(') {
781 			read();
782 			token = Token.OPEN_PAREN;
783 			return token;
784 		}
785 		if (c == ')') {
786 			read();
787 			token = Token.CLOSE_PAREN;
788 			return token;
789 		}
790 		if (c == '{') {
791 			read();
792 			skipWhitespaces();
793 			token = Token.OPEN_BRACE;
794 			return token;
795 		}
796 		if (c == '}') {
797 			read();
798 			token = Token.CLOSE_BRACE;
799 			return token;
800 		}
801 		if (c == '[') {
802 			read();
803 			token = Token.OPEN_BRACKET;
804 			return token;
805 		}
806 		if (c == ']') {
807 			read();
808 			token = Token.CLOSE_BRACKET;
809 			return token;
810 		}
811 		if (c == '$') {
812 			read();
813 			token = Token.DOLLAR;
814 			return token;
815 		}
816 		if (c == '@') {
817 			if (posix) {
818 				throw lexerException("gawk @ syntax is not supported in POSIX mode.");
819 			}
820 			read();
821 			if (c == '/') {
822 				read();
823 				readRegexp();
824 				token = Token.TYPED_REGEXP;
825 				return token;
826 			}
827 			if (Character.isJavaIdentifierStart(c)) {
828 				while (Character.isJavaIdentifierPart(c)) {
829 					read();
830 				}
831 				if (c == ':') {
832 					read();
833 					if (c != ':') {
834 						throw lexerException("Namespace separator must be two colons (::).");
835 					}
836 					read();
837 					if (!Character.isJavaIdentifierStart(c)) {
838 						throw lexerException("A namespace-qualified name requires an identifier after ::.");
839 					}
840 					read();
841 					while (Character.isJavaIdentifierPart(c)) {
842 						read();
843 					}
844 				}
845 				String atWord = text.toString();
846 				if ("@include".equals(atWord)) {
847 					token = Token.DIRECTIVE_INCLUDE;
848 					return token;
849 				}
850 				if ("@namespace".equals(atWord)) {
851 					token = Token.DIRECTIVE_NAMESPACE;
852 					return token;
853 				}
854 				if ("@load".equals(atWord)) {
855 					token = Token.DIRECTIVE_UNSUPPORTED;
856 					return token;
857 				}
858 				pendingIndirectIdentifier = atWord.substring(1);
859 				validateIndirectIdentifier(pendingIndirectIdentifier);
860 				token = Token.INDIRECT;
861 				return token;
862 			}
863 			token = Token.INDIRECT;
864 			return token;
865 		}
866 		if (c == '~') {
867 			read();
868 			token = Token.MATCHES;
869 			return token;
870 		}
871 		if (c == '?') {
872 			read();
873 			skipWhitespaces();
874 			token = Token.QUESTION_MARK;
875 			return token;
876 		}
877 		if (c == ':') {
878 			read();
879 			skipWhitespaces();
880 			token = Token.COLON;
881 			return token;
882 		}
883 		if (c == '&') {
884 			read();
885 			if (c == '&') {
886 				read();
887 				skipWhitespaces();
888 				token = Token.AND;
889 				return token;
890 			}
891 			throw lexerException("use && for logical and");
892 		}
893 		if (c == '|') {
894 			read();
895 			if (c == '|') {
896 				read();
897 				skipWhitespaces();
898 				token = Token.OR;
899 				return token;
900 			}
901 			token = Token.PIPE;
902 			return token;
903 		}
904 		if (c == '=') {
905 			read();
906 			if (c == '=') {
907 				read();
908 				token = Token.EQ;
909 				return token;
910 			}
911 			token = Token.EQUALS;
912 			return token;
913 		}
914 		if (c == '+') {
915 			read();
916 			if (c == '=') {
917 				read();
918 				token = Token.PLUS_EQ;
919 				return token;
920 			} else if (c == '+') {
921 				read();
922 				token = Token.INC;
923 				return token;
924 			}
925 			token = Token.PLUS;
926 			return token;
927 		}
928 		if (c == '-') {
929 			read();
930 			if (c == '=') {
931 				read();
932 				token = Token.MINUS_EQ;
933 				return token;
934 			} else if (c == '-') {
935 				read();
936 				token = Token.DEC;
937 				return token;
938 			}
939 			token = Token.MINUS;
940 			return token;
941 		}
942 		if (c == '*') {
943 			read();
944 			if (c == '=') {
945 				read();
946 				token = Token.MULT_EQ;
947 				return token;
948 			} else if (c == '*') {
949 				read();
950 				if (c == '=') {
951 					read();
952 					token = Token.POW_EQ;
953 					return token;
954 				}
955 				token = Token.POW;
956 				return token;
957 			}
958 			token = Token.MULT;
959 			return token;
960 		}
961 		if (c == '/') {
962 			read();
963 			if (c == '=') {
964 				read();
965 				token = Token.DIV_EQ;
966 				return token;
967 			}
968 			token = Token.DIVIDE;
969 			return token;
970 		}
971 		if (c == '%') {
972 			read();
973 			if (c == '=') {
974 				read();
975 				token = Token.MOD_EQ;
976 				return token;
977 			}
978 			token = Token.MOD;
979 			return token;
980 		}
981 		if (c == '^') {
982 			read();
983 			if (c == '=') {
984 				read();
985 				token = Token.POW_EQ;
986 				return token;
987 			}
988 			token = Token.POW;
989 			return token;
990 		}
991 		if (c == '>') {
992 			read();
993 			if (c == '=') {
994 				read();
995 				token = Token.GE;
996 				return token;
997 			} else if (c == '>') {
998 				read();
999 				token = Token.APPEND;
1000 				return token;
1001 			}
1002 			token = Token.GT;
1003 			return token;
1004 		}
1005 		if (c == '<') {
1006 			read();
1007 			if (c == '=') {
1008 				read();
1009 				token = Token.LE;
1010 				return token;
1011 			}
1012 			token = Token.LT;
1013 			return token;
1014 		}
1015 		if (c == '!') {
1016 			read();
1017 			if (c == '=') {
1018 				read();
1019 				token = Token.NE;
1020 				return token;
1021 			} else if (c == '~') {
1022 				read();
1023 				token = Token.NOT_MATCHES;
1024 				return token;
1025 			}
1026 			token = Token.NOT;
1027 			return token;
1028 		}
1029 
1030 		if (c == '.') {
1031 			// double!
1032 			read();
1033 			boolean hit = false;
1034 			while (c > 0 && Character.isDigit(c)) {
1035 				hit = true;
1036 				read();
1037 			}
1038 			if (!hit) {
1039 				throw lexerException("Decimal point encountered with no values on either side.");
1040 			}
1041 			readExponent();
1042 			token = Token.DOUBLE;
1043 			return token;
1044 		}
1045 
1046 		if (Character.isDigit(c)) {
1047 			// integer or double.
1048 			read();
1049 			while (c > 0) {
1050 				if (c == '.') {
1051 					// double!
1052 					read();
1053 					while (c > 0 && Character.isDigit(c)) {
1054 						read();
1055 					}
1056 					readExponent();
1057 					token = Token.DOUBLE;
1058 					return token;
1059 				} else if (Character.isDigit(c)) {
1060 					// integer or double.
1061 					read();
1062 				} else {
1063 					break;
1064 				}
1065 			}
1066 			if (readExponent()) {
1067 				token = Token.DOUBLE;
1068 				return token;
1069 			}
1070 			// integer, only
1071 			token = Token.INTEGER;
1072 			return token;
1073 		}
1074 
1075 		if (Character.isJavaIdentifierStart(c)) {
1076 			read();
1077 			while (Character.isJavaIdentifierPart(c)) {
1078 				read();
1079 			}
1080 			if (c == ':') {
1081 				read();
1082 				if (c != ':') {
1083 					text.setLength(text.length() - 1);
1084 					pendingColon = true;
1085 				} else {
1086 					if (posix) {
1087 						throw lexerException("gawk namespace syntax is not supported in POSIX mode.");
1088 					}
1089 					read();
1090 					if (!Character.isJavaIdentifierStart(c)) {
1091 						throw lexerException("A namespace-qualified name requires an identifier after ::.");
1092 					}
1093 					read();
1094 					while (Character.isJavaIdentifierPart(c)) {
1095 						read();
1096 					}
1097 					if (c == ':') {
1098 						read();
1099 						if (c == ':') {
1100 							throw lexerException("A namespace-qualified name may contain only one :: separator.");
1101 						}
1102 						text.setLength(text.length() - 1);
1103 						pendingColon = true;
1104 					}
1105 				}
1106 			}
1107 			// check for certain keywords
1108 			// extensions override built-in stuff
1109 			String sourceIdentifier = text.toString();
1110 			int namespaceSeparator = sourceIdentifier.indexOf("::");
1111 			if (namespaceSeparator >= 0) {
1112 				String namespaceComponent = sourceIdentifier.substring(namespaceSeparator + 2);
1113 				if (KEYWORDS.containsKey(namespaceComponent) || BuiltinFunction.of(namespaceComponent) != null) {
1114 					throw lexerException("Reserved word cannot be used after a namespace separator: " + sourceIdentifier);
1115 				}
1116 			}
1117 			String lookupIdentifier = awkNamespaceComponent(sourceIdentifier);
1118 			boolean awkNamespaceIdentifier = isAwkNamespaceIdentifier(sourceIdentifier);
1119 			boolean unqualifiedIdentifier = namespaceSeparator < 0;
1120 			if (awkNamespaceIdentifier && extensions.get(lookupIdentifier) != null) {
1121 				text.setLength(0);
1122 				text.append(lookupIdentifier);
1123 				token = Token.EXTENSION;
1124 				return token;
1125 			}
1126 			Token kwToken = KEYWORDS.get(sourceIdentifier);
1127 			if (kwToken != null && !isDisabledKeyword(kwToken)) {
1128 				token = kwToken;
1129 				return token;
1130 			}
1131 			if ((unqualifiedIdentifier || awkNamespaceIdentifier) && BuiltinFunction.of(lookupIdentifier) != null) {
1132 				text.setLength(0);
1133 				text.append(lookupIdentifier);
1134 				token = Token.BUILTIN_FUNC_NAME;
1135 				return token;
1136 			}
1137 			if (c == '(' && !pendingColon) {
1138 				token = Token.FUNC_ID;
1139 				return token;
1140 			} else {
1141 				token = Token.ID;
1142 				return token;
1143 			}
1144 		}
1145 
1146 		if (c == ';') {
1147 			read();
1148 			while (c == ' ' || c == '\t' || c == '\n' || c == '#') {
1149 				if (c == '\n') {
1150 					break;
1151 				}
1152 				if (c == '#') {
1153 					while (c >= 0 && c != '\n') {
1154 						read();
1155 					}
1156 					if (c == '\n') {
1157 						read();
1158 					}
1159 				} else {
1160 					read();
1161 				}
1162 			}
1163 			token = Token.SEMICOLON;
1164 			return token;
1165 		}
1166 
1167 		if (c == '\n') {
1168 			read();
1169 			while (c == ' ' || c == '\t' || c == '#' || c == '\n') {
1170 				if (c == '#') {
1171 					while (c >= 0 && c != '\n') {
1172 						read();
1173 					}
1174 				}
1175 				read();
1176 			}
1177 			token = Token.NEWLINE;
1178 			return token;
1179 		}
1180 
1181 		if (c == '"') {
1182 			// string
1183 			read();
1184 			readString();
1185 			token = Token.STRING;
1186 			return token;
1187 		}
1188 
1189 		/*
1190 		 * if (c == '\\') {
1191 		 * c = reader.read();
1192 		 * // completely bypass \r's
1193 		 * while(c == '\r') c = reader.read();
1194 		 * if (c<0)
1195 		 * chr=0; // eof
1196 		 * else
1197 		 * chr=c;
1198 		 * }
1199 		 */
1200 
1201 		throw lexerException("Invalid character (" + c + "): " + ((char) c));
1202 	}
1203 
1204 	// SUPPORTING FUNCTIONS/METHODS
1205 	private void terminator() throws IOException {
1206 		// like optTerminator, except error if no terminator was found
1207 		if (!optTerminator()) {
1208 			throw parserException("Expecting statement terminator. Got " + token.name() + ": " + text);
1209 		}
1210 	}
1211 
1212 	private boolean optTerminator() throws IOException {
1213 		if (optNewline()) {
1214 			return true;
1215 		} else if (token == Token.EOF || token == Token.CLOSE_BRACE) {
1216 			return true; // do nothing
1217 		} else if (token == Token.SEMICOLON) {
1218 			lexer();
1219 			return true;
1220 		} else {
1221 			// no terminator consumed
1222 			return false;
1223 		}
1224 	}
1225 
1226 	private boolean optNewline() throws IOException {
1227 		if (token == Token.NEWLINE) {
1228 			lexer();
1229 			return true;
1230 		} else {
1231 			return false;
1232 		}
1233 	}
1234 
1235 	// RECURSIVE DECENT PARSER:
1236 	// CHECKSTYLE.OFF: MethodName
1237 	// SCRIPT : \n [RULE_LIST] Token.EOF
1238 	AST SCRIPT() throws IOException {
1239 		AST rl;
1240 		if (token != Token.EOF) {
1241 			rl = RULE_LIST();
1242 		} else {
1243 			rl = null;
1244 		}
1245 		lexer(Token.EOF);
1246 		return rl;
1247 	}
1248 
1249 	// EXPRESSION_TO_EVALUATE: [TERNARY_EXPRESSION] Token.EOF
1250 	// Used to parse simple expressions to evaluate instead of full scripts
1251 	AST EXPRESSION_TO_EVALUATE() throws IOException {
1252 		AST exprAst = token != Token.EOF ? TERNARY_EXPRESSION(null, true, false, true) : null;
1253 		lexer(Token.EOF);
1254 		return new ExpressionToEvaluateAst(exprAst);
1255 	}
1256 
1257 	// RULE_LIST : \n [ ( RULE | FUNCTION terminator ) optTerminator RULE_LIST ]
1258 	AST RULE_LIST() throws IOException {
1259 		optNewline();
1260 		AST ruleOrFunction = null;
1261 		if (token == Token.DIRECTIVE_INCLUDE) {
1262 			INCLUDE_DIRECTIVE();
1263 			return RULE_LIST();
1264 		} else if (token == Token.DIRECTIVE_NAMESPACE) {
1265 			NAMESPACE_DIRECTIVE();
1266 			return RULE_LIST();
1267 		} else if (token == Token.DIRECTIVE_UNSUPPORTED) {
1268 			throw parserException("Unsupported gawk directive: " + text);
1269 		} else if (token == Token.KW_FUNCTION) {
1270 			ruleOrFunction = FUNCTION();
1271 		} else if (token != Token.EOF) {
1272 			ruleOrFunction = RULE();
1273 		} else {
1274 			return null;
1275 		}
1276 		optTerminator(); // newline or ; (maybe)
1277 		return new RuleListAst(ruleOrFunction, RULE_LIST());
1278 	}
1279 
1280 	private void NAMESPACE_DIRECTIVE() throws IOException {
1281 		lexer();
1282 		if (token != Token.STRING) {
1283 			throw parserException("@namespace requires a quoted namespace name.");
1284 		}
1285 		String namespace = string.toString();
1286 		validateNamespace(namespace);
1287 		currentNamespace = namespace;
1288 		lexer();
1289 		terminator();
1290 	}
1291 
1292 	private void INCLUDE_DIRECTIVE() throws IOException {
1293 		lexer();
1294 		if (token != Token.STRING) {
1295 			throw parserException("@include requires a quoted file name.");
1296 		}
1297 		if (!sourceIncludeAllowed) {
1298 			throw new AwkSandboxException("@include is disabled in sandbox mode");
1299 		}
1300 		String includeName = string.toString();
1301 		boolean includeTerminatedByEndOfSource = validateIncludeTerminator();
1302 		Path includePath = resolveIncludePath(includeName);
1303 		if (topLevelSourcePaths.contains(includePath)) {
1304 			throw parserException("Cannot include a top-level program source: " + includeName);
1305 		}
1306 		if (!includedSourcePaths.add(includePath)) {
1307 			lexer();
1308 			if (!includeTerminatedByEndOfSource) {
1309 				terminator();
1310 			}
1311 			return;
1312 		}
1313 		includedSourceStack.push(new SourceState(currentScriptSource, reader, c, currentNamespace));
1314 		currentScriptSource = new ScriptSource(
1315 				includePath.toString(),
1316 				Files.newBufferedReader(includePath, StandardCharsets.UTF_8));
1317 		reader = new LineNumberReader(currentScriptSource.getReader());
1318 		currentNamespace = "awk";
1319 		c = reader.read();
1320 		while (c == '\r') {
1321 			c = reader.read();
1322 		}
1323 		advancePastEndOfSource();
1324 		lexer();
1325 	}
1326 
1327 	private boolean validateIncludeTerminator() throws IOException {
1328 		while (c == ' ' || c == '\t') {
1329 			read();
1330 		}
1331 		if (c == '#') {
1332 			while (c >= 0 && c != '\n') {
1333 				read();
1334 			}
1335 		}
1336 		if (c >= 0 && c != '\n' && c != ';') {
1337 			throw parserException("@include must be followed by a newline, semicolon, or end of file.");
1338 		}
1339 		return c < 0;
1340 	}
1341 
1342 	private void validateNamespace(String namespace) {
1343 		if (namespace == null || namespace.isEmpty() || !Character.isJavaIdentifierStart(namespace.charAt(0))) {
1344 			throw parserException("Invalid gawk namespace name: " + namespace);
1345 		}
1346 		for (int i = 1; i < namespace.length(); i++) {
1347 			if (!Character.isJavaIdentifierPart(namespace.charAt(i))) {
1348 				throw parserException("Invalid gawk namespace name: " + namespace);
1349 			}
1350 		}
1351 		if (KEYWORDS.containsKey(namespace) || BuiltinFunction.of(namespace) != null || extensions.containsKey(namespace)) {
1352 			throw parserException("Reserved identifier cannot be used as a gawk namespace: " + namespace);
1353 		}
1354 	}
1355 
1356 	private void validateIndirectIdentifier(String identifier) throws LexerException {
1357 		int separator = identifier.indexOf("::");
1358 		String namespace = separator < 0 ? "awk" : identifier.substring(0, separator);
1359 		String component = separator < 0 ? identifier : identifier.substring(separator + 2);
1360 		if (KEYWORDS.containsKey(component)
1361 				|| BuiltinFunction.of(component) != null
1362 				|| ("awk".equals(namespace) && extensions.containsKey(component))) {
1363 			throw lexerException("Reserved identifier cannot be used as an indirect-call selector: " + identifier);
1364 		}
1365 	}
1366 
1367 	private Path resolveIncludePath(String includeName) {
1368 		Path requested = Paths.get(includeName);
1369 		List<Path> candidates = new ArrayList<Path>();
1370 		if (requested.isAbsolute()) {
1371 			candidates.add(requested);
1372 		} else {
1373 			if (!ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT.equals(currentScriptSource.getDescription())) {
1374 				try {
1375 					Path sourcePath = Paths.get(currentScriptSource.getDescription());
1376 					Path parent = sourcePath.toAbsolutePath().normalize().getParent();
1377 					if (parent != null) {
1378 						candidates.add(parent.resolve(requested));
1379 					}
1380 				} catch (InvalidPathException ignored) {
1381 					// Reader-backed ScriptSource values may use a descriptive
1382 					// label rather than a file path.
1383 				}
1384 			}
1385 			String awkPath = System.getenv("AWKPATH");
1386 			if (awkPath != null) {
1387 				for (String entry : awkPath.split(java.util.regex.Pattern.quote(File.pathSeparator), -1)) {
1388 					candidates.add(Paths.get(entry.isEmpty() ? "." : entry).resolve(requested));
1389 				}
1390 			}
1391 			candidates.add(requested);
1392 		}
1393 		for (Path candidate : candidates) {
1394 			Path normalized = candidate.toAbsolutePath().normalize();
1395 			if (Files.isRegularFile(normalized)) {
1396 				try {
1397 					return normalized.toRealPath();
1398 				} catch (IOException ignored) {
1399 					// The candidate may have disappeared between the existence
1400 					// check and canonicalization; continue searching AWKPATH.
1401 				}
1402 			}
1403 		}
1404 		throw parserException("Cannot find @include file: " + includeName);
1405 	}
1406 
1407 	// FUNCTION: function functionName( [FORMAL_PARAM_LIST] ) STATEMENT_LIST
1408 	AST FUNCTION() throws IOException {
1409 		expectKeyword("function");
1410 		String functionName;
1411 		if (token == Token.FUNC_ID || token == Token.ID) {
1412 			functionName = qualifyGlobalIdentifier(text.toString());
1413 			lexer();
1414 		} else {
1415 			throw parserException("Expecting function name. Got " + token.name() + ": " + text);
1416 		}
1417 		symbolTable.setFunctionName(functionName);
1418 		lexer(Token.OPEN_PAREN);
1419 		AST formalParamList;
1420 		if (token == Token.CLOSE_PAREN) {
1421 			formalParamList = null;
1422 		} else {
1423 			formalParamList = FORMAL_PARAM_LIST(functionName);
1424 		}
1425 		lexer(Token.CLOSE_PAREN);
1426 		optNewline();
1427 
1428 		lexer(Token.OPEN_BRACE);
1429 		AST functionBlock = STATEMENT_LIST();
1430 		lexer(Token.CLOSE_BRACE);
1431 		symbolTable.clearFunctionName(functionName);
1432 		return symbolTable.addFunctionDef(functionName, formalParamList, functionBlock);
1433 	}
1434 
1435 	// FORMAT_PARAM_LIST:
1436 	AST FORMAL_PARAM_LIST(String functionName) throws IOException {
1437 		if (token == Token.ID) {
1438 			String id = text.toString();
1439 			symbolTable.addFunctionParameter(functionName, id);
1440 			lexer();
1441 			if (token == Token.COMMA) {
1442 				lexer();
1443 				optNewline();
1444 				AST rest = FORMAL_PARAM_LIST(functionName);
1445 				if (rest == null) {
1446 					throw parserException("Cannot terminate a formal parameter list with a comma.");
1447 				} else {
1448 					return new FunctionDefParamListAst(id, rest);
1449 				}
1450 			} else {
1451 				return new FunctionDefParamListAst(id, null);
1452 			}
1453 		} else {
1454 			return null;
1455 		}
1456 	}
1457 
1458 	// RULE : [ASSIGNMENT_EXPRESSION] [ { STATEMENT_LIST } ]
1459 	AST RULE() throws IOException {
1460 		AST optExpr;
1461 		AST optStmts;
1462 		if (token == Token.KW_BEGIN) {
1463 			lexer();
1464 			optExpr = symbolTable.addBEGIN();
1465 		} else if (token == Token.KW_END) {
1466 			lexer();
1467 			optExpr = symbolTable.addEND();
1468 		} else if (token == Token.KW_BEGINFILE) {
1469 			lexer();
1470 			optExpr = symbolTable.addBEGINFILE();
1471 		} else if (token == Token.KW_ENDFILE) {
1472 			lexer();
1473 			optExpr = symbolTable.addENDFILE();
1474 		} else if (token != Token.OPEN_BRACE && token != Token.SEMICOLON && token != Token.NEWLINE && token != Token.EOF) {
1475 			// true = allow comparators, allow IN keyword, do Token.NOT allow multidim indices expressions
1476 			optExpr = ASSIGNMENT_EXPRESSION(null, true, true, false);
1477 			// for ranges, like conditionStart, conditionEnd
1478 			if (token == Token.COMMA) {
1479 				lexer();
1480 				optNewline();
1481 				// true = allow comparators, allow IN keyword, do Token.NOT allow multidim indices expressions
1482 				optExpr = new ConditionPairAst(optExpr, ASSIGNMENT_EXPRESSION(null, true, true, false));
1483 			}
1484 		} else {
1485 			optExpr = null;
1486 		}
1487 		if (token == Token.OPEN_BRACE) {
1488 			lexer();
1489 			optStmts = STATEMENT_LIST();
1490 			lexer(Token.CLOSE_BRACE);
1491 		} else {
1492 			optStmts = null;
1493 		}
1494 		return new RuleAst(optExpr, optStmts);
1495 	}
1496 
1497 	// STATEMENT_LIST : [ STATEMENT_BLOCK|STATEMENT STATEMENT_LIST ]
1498 	private AST STATEMENT_LIST() throws IOException {
1499 		// statement lists can only live within curly brackets (braces)
1500 		optNewline();
1501 		if (token == Token.CLOSE_BRACE || token == Token.EOF) {
1502 			return null;
1503 		}
1504 		AST stmt;
1505 		if (token == Token.OPEN_BRACE) {
1506 			lexer();
1507 			stmt = STATEMENT_LIST();
1508 			lexer(Token.CLOSE_BRACE);
1509 		} else {
1510 			if (token == Token.SEMICOLON) {
1511 				// an empty statement (;)
1512 				// do not polute the syntax tree with nulls in this case
1513 				// just return the next statement (recursively)
1514 				lexer();
1515 				return STATEMENT_LIST();
1516 			} else {
1517 				stmt = STATEMENT();
1518 			}
1519 		}
1520 
1521 		AST rest = STATEMENT_LIST();
1522 		if (rest == null) {
1523 			return stmt;
1524 		} else if (stmt == null) {
1525 			return rest;
1526 		} else {
1527 			return new StatementListAst(stmt, rest);
1528 		}
1529 	}
1530 
1531 	/**
1532 	 * Parse a (possibly comma-separated) list of ASSIGNMENT_EXPRESSIONs.
1533 	 *
1534 	 * @param allowComparisons
1535 	 *        – true ⇒ treat ‘>’ and ‘<’ as comparison operators
1536 	 *        – false ⇒ treat ‘>’ and ‘<’ as redirection tokens (break out)
1537 	 * @param allowInKeyword
1538 	 *        – true ⇒ allow the “in” keyword inside expressions
1539 	 *        – false ⇒ disallow “in”
1540 	 */
1541 	AST EXPRESSION_LIST(boolean allowComparisons, boolean allowInKeyword) throws IOException {
1542 		// Captured before parsing: the list node's line is where its first
1543 		// expression starts, so a parenthesized group rebuilt into a
1544 		// multi-dimensional subscript reports errors there.
1545 		int lineNo = currentSourceLineNumber();
1546 
1547 		// 1) Parse exactly one assignment expression.
1548 		// Passing `allowComparisons` will decide if ‘>’/’<’ become comparisons or redirectors.
1549 		AST expr = ASSIGNMENT_EXPRESSION(null, allowComparisons, allowInKeyword, /* allowMultidim= */ false);
1550 
1551 		// 2) If the next token is a comma, consume it and build the rest of the list.
1552 		// This supports both regular function calls and print/printf argument lists.
1553 		if (token == Token.COMMA) {
1554 			lexer(); // consume ','
1555 			optNewline(); // allow newline after comma (AWK style)
1556 
1557 			AST rest = EXPRESSION_LIST(allowComparisons, allowInKeyword);
1558 			return new FunctionCallParamListAst(lineNo, expr, rest);
1559 		}
1560 
1561 		// 3) No comma ⇒ this single expression is a one‐element list.
1562 		return new FunctionCallParamListAst(lineNo, expr, null);
1563 	}
1564 
1565 	private AST ASSIGNMENT_EXPRESSION(
1566 			AST left,
1567 			boolean allowComparison,
1568 			boolean allowInKeyword,
1569 			boolean allowMultidimIndices)
1570 			throws IOException {
1571 		// Captured before parsing: if the expression turns out to be the first
1572 		// component of a multi-dimensional subscript group, the group's line is
1573 		// where this component starts.
1574 		int startLineNo = currentSourceLineNumber();
1575 		AST ternaryExpression = TERNARY_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1576 		AST result = ternaryExpression;
1577 		if (token == Token.EQUALS
1578 				|| token == Token.PLUS_EQ
1579 				|| token == Token.MINUS_EQ
1580 				|| token == Token.MULT_EQ
1581 				|| token == Token.DIV_EQ
1582 				|| token == Token.MOD_EQ
1583 				|| token == Token.POW_EQ) {
1584 			Token op = token;
1585 			String txt = text.toString();
1586 			lexer();
1587 			// An assignment RHS is a single expression: a following comma belongs to
1588 			// the enclosing grouping, as in ((y=1, 2) in a), where the group elements
1589 			// are (y=1) and (2)
1590 			AST assignmentExpression = ASSIGNMENT_EXPRESSION(null, allowComparison, allowInKeyword, false);
1591 			result = new AssignmentExpressionAst(ternaryExpression, op, txt, assignmentExpression);
1592 		}
1593 		// ASSIGNMENT_EXPRESSION [, ASSIGNMENT_EXPRESSION] !!!ONLY IF!!! allowMultidimIndices is true
1594 		// allowMultidimIndices is set to true when we need (1,2,3,4) expressions to collapse into an array index
1595 		// expression (converts 1,2,3,4 to 1 SUBSEP 2 SUBSEP 3 SUBSEP 4) after an open parenthesis (grouping)
1596 		// expression starter
1597 		if (allowMultidimIndices && token == Token.COMMA) {
1598 			lexer();
1599 			optNewline();
1600 			int restLineNo = currentSourceLineNumber();
1601 			AST rest = ASSIGNMENT_EXPRESSION(null, allowComparison, allowInKeyword, true);
1602 			if (rest instanceof ArrayIndexAst) {
1603 				return new ArrayIndexAst(startLineNo, result, rest);
1604 			}
1605 			return new ArrayIndexAst(startLineNo, result, new ArrayIndexAst(restLineNo, rest, null));
1606 		}
1607 		return result;
1608 	}
1609 
1610 	// TERNARY_EXPRESSION = LOGICAL_OR_EXPRESSION [ ? TERNARY_EXPRESSION : TERNARY_EXPRESSION ]
1611 	private AST TERNARY_EXPRESSION(
1612 			AST left,
1613 			boolean allowComparison,
1614 			boolean allowInKeyword,
1615 			boolean allowMultidimIndices)
1616 			throws IOException {
1617 		AST condition = LOGICAL_OR_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1618 		if (token == Token.QUESTION_MARK) {
1619 			lexer();
1620 			AST trueBlock = TERNARY_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
1621 			lexer(Token.COLON);
1622 			AST falseBlock = TERNARY_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
1623 			return new TernaryExpressionAst(condition, trueBlock, falseBlock);
1624 		}
1625 		return condition;
1626 	}
1627 
1628 	// LOGICAL_OR_EXPRESSION = LOGICAL_AND_EXPRESSION [ || LOGICAL_OR_EXPRESSION ]
1629 	private AST LOGICAL_OR_EXPRESSION(
1630 			AST left,
1631 			boolean allowComparison,
1632 			boolean allowInKeyword,
1633 			boolean allowMultidimIndices)
1634 			throws IOException {
1635 		AST result = LOGICAL_AND_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1636 		while (token == Token.OR) {
1637 			Token op = token;
1638 			String txt = text.toString();
1639 			lexer();
1640 			AST rhs = LOGICAL_OR_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
1641 			result = new LogicalExpressionAst(result, op, txt, rhs);
1642 		}
1643 		return result;
1644 	}
1645 
1646 	// LOGICAL_AND_EXPRESSION = IN_EXPRESSION [ && LOGICAL_AND_EXPRESSION ]
1647 	private AST LOGICAL_AND_EXPRESSION(
1648 			AST left,
1649 			boolean allowComparison,
1650 			boolean allowInKeyword,
1651 			boolean allowMultidimIndices)
1652 			throws IOException {
1653 		AST result = IN_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1654 		while (token == Token.AND) {
1655 			Token op = token;
1656 			String txt = text.toString();
1657 			lexer();
1658 			AST rhs = LOGICAL_AND_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
1659 			result = new LogicalExpressionAst(result, op, txt, rhs);
1660 		}
1661 		return result;
1662 	}
1663 
1664 	// IN_EXPRESSION = MATCHING_EXPRESSION [ IN_EXPRESSION ]
1665 	// allowInKeyword is set false while parsing the first expression within
1666 	// a for() statement (because it could be "for (key in arr)", and this
1667 	// production will consume and the for statement will never have a chance
1668 	// of processing it
1669 	// all other times, it is true
1670 	private AST IN_EXPRESSION(AST left, boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
1671 			throws IOException {
1672 		AST result = MATCHING_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1673 		if (allowInKeyword && token == Token.KW_IN) {
1674 			lexer();
1675 			result = new InExpressionAst(result, IN_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices));
1676 		}
1677 		return result;
1678 	}
1679 
1680 	// MATCHING_EXPRESSION = COMPARISON_EXPRESSION [ (~,!~) MATCHING_EXPRESSION ]
1681 	private AST MATCHING_EXPRESSION(
1682 			AST left,
1683 			boolean allowComparison,
1684 			boolean allowInKeyword,
1685 			boolean allowMultidimIndices)
1686 			throws IOException {
1687 		AST result = COMPARISON_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1688 		while (token == Token.MATCHES || token == Token.NOT_MATCHES) {
1689 			Token op = token;
1690 			String txt = text.toString();
1691 			lexer();
1692 			AST rhs = MATCHING_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
1693 			result = new ComparisonExpressionAst(result, op, txt, rhs);
1694 		}
1695 		return result;
1696 	}
1697 
1698 	// COMPARISON_EXPRESSION = CONCAT_EXPRESSION [ (==,>,>=,<,<=,!=,|) COMPARISON_EXPRESSION ]
1699 	// allowComparison is set false when within a print/printf statement;
1700 	// all other times it is set true
1701 	private AST COMPARISON_EXPRESSION(
1702 			AST left,
1703 			boolean allowComparison,
1704 			boolean allowInKeyword,
1705 			boolean allowMultidimIndices)
1706 			throws IOException {
1707 		AST result = CONCAT_EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1708 		if (token == Token.EQ
1709 				|| token == Token.GE
1710 				|| token == Token.LT
1711 				|| token == Token.LE
1712 				|| token == Token.NE
1713 				|| (token == Token.GT && allowComparison)) {
1714 			Token op = token;
1715 			String txt = text.toString();
1716 			lexer();
1717 			AST rhs = COMPARISON_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices);
1718 			return new ComparisonExpressionAst(result, op, txt, rhs);
1719 		} else if (allowComparison && token == Token.PIPE) {
1720 			lexer();
1721 			return GETLINE_EXPRESSION(result, allowComparison, allowInKeyword);
1722 		}
1723 
1724 		return result;
1725 	}
1726 
1727 	// CONCAT_EXPRESSION = EXPRESSION [ CONCAT_EXPRESSION ]
1728 	private AST CONCAT_EXPRESSION(AST left, boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
1729 			throws IOException {
1730 		AST result = EXPRESSION(left, allowComparison, allowInKeyword, allowMultidimIndices);
1731 		if (token == Token.INTEGER
1732 				|| token == Token.DOUBLE
1733 				|| token == Token.OPEN_PAREN
1734 				|| token == Token.FUNC_ID
1735 				|| token == Token.INC
1736 				|| token == Token.DEC
1737 				|| token == Token.ID
1738 				|| token == Token.STRING
1739 				|| token == Token.DOLLAR
1740 				|| token == Token.BUILTIN_FUNC_NAME
1741 				|| token == Token.EXTENSION) {
1742 			return new ConcatExpressionAst(
1743 					result,
1744 					CONCAT_EXPRESSION(null, allowComparison, allowInKeyword, allowMultidimIndices));
1745 		}
1746 		return result;
1747 	}
1748 
1749 	// EXPRESSION : TERM [ (+|-) EXPRESSION ]
1750 	private AST EXPRESSION(AST left, boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
1751 			throws IOException {
1752 		AST result = TERM(left, allowComparison, allowInKeyword, allowMultidimIndices);
1753 		while (token == Token.PLUS || token == Token.MINUS) {
1754 			Token op = token;
1755 			String txt = text.toString();
1756 			lexer();
1757 			AST nextTerm = TERM(null, allowComparison, allowInKeyword, allowMultidimIndices);
1758 			result = new BinaryExpressionAst(result, op, txt, nextTerm);
1759 		}
1760 		return result;
1761 	}
1762 
1763 	// TERM : UNARY_FACTOR [ (*|/|%) TERM ]
1764 	private AST TERM(AST left, boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
1765 			throws IOException {
1766 		AST result = (left == null) ? UNARY_FACTOR(allowComparison, allowInKeyword, allowMultidimIndices) : left;
1767 		while (token == Token.MULT || token == Token.DIVIDE || token == Token.MOD) {
1768 			Token op = token;
1769 			String txt = text.toString();
1770 			lexer();
1771 			AST nextUnaryFactor = UNARY_FACTOR(allowComparison, allowInKeyword, allowMultidimIndices);
1772 			result = new BinaryExpressionAst(result, op, txt, nextUnaryFactor);
1773 		}
1774 		return result;
1775 	}
1776 
1777 	// UNARY_FACTOR : [ ! | - | + ] POWER_FACTOR
1778 	AST UNARY_FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices) throws IOException {
1779 		if (token == Token.NOT) {
1780 			lexer();
1781 			return new NotExpressionAst(POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices));
1782 		} else if (token == Token.MINUS) {
1783 			lexer();
1784 			return new NegativeExpressionAst(POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices));
1785 		} else if (token == Token.PLUS) {
1786 			lexer();
1787 			return new UnaryPlusExpressionAst(POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices));
1788 		} else {
1789 			return POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices);
1790 		}
1791 	}
1792 
1793 	// POWER_FACTOR : FACTOR_FOR_INCDEC [ ^ POWER_FACTOR ]
1794 	private AST POWER_FACTOR(AST left, boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
1795 			throws IOException {
1796 		AST result = (left == null) ? FACTOR_FOR_INCDEC(allowComparison, allowInKeyword, allowMultidimIndices) : left;
1797 		if (token == Token.POW) {
1798 			Token op = token;
1799 			String txt = text.toString();
1800 			lexer();
1801 			AST rhs = POWER_FACTOR(null, allowComparison, allowInKeyword, allowMultidimIndices);
1802 			return new BinaryExpressionAst(result, op, txt, rhs);
1803 		}
1804 		return result;
1805 	}
1806 
1807 	// according to the spec, pre/post inc can occur
1808 	// only on lvalues, which are NAMES (IDs), array,
1809 	// or field references
1810 	private boolean isLvalue(AST ast) {
1811 		return (ast instanceof IDAst) || (ast instanceof ArrayReferenceAst) || (ast instanceof DollarExpressionAst);
1812 	}
1813 
1814 	AST FACTOR_FOR_INCDEC(boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices)
1815 			throws IOException {
1816 		boolean preInc = false;
1817 		boolean preDec = false;
1818 		boolean postInc = false;
1819 		boolean postDec = false;
1820 		if (token == Token.INC) {
1821 			preInc = true;
1822 			lexer();
1823 		} else if (token == Token.DEC) {
1824 			preDec = true;
1825 			lexer();
1826 		}
1827 
1828 		AST factorAst = FACTOR(allowComparison, allowInKeyword, allowMultidimIndices);
1829 
1830 		if ((preInc || preDec) && !isLvalue(factorAst)) {
1831 			throw parserException("Cannot pre inc/dec a non-lvalue");
1832 		}
1833 
1834 		// only do post ops if:
1835 		// - factorAst is an lvalue
1836 		// - pre ops were not encountered
1837 		if (isLvalue(factorAst) && !preInc && !preDec) {
1838 			if (token == Token.INC) {
1839 				postInc = true;
1840 				lexer();
1841 			} else if (token == Token.DEC) {
1842 				postDec = true;
1843 				lexer();
1844 			}
1845 		}
1846 
1847 		if ((preInc || preDec) && (postInc || postDec)) {
1848 			throw parserException("Cannot do pre inc/dec Token.AND post inc/dec.");
1849 		}
1850 
1851 		if (preInc) {
1852 			return new PreIncAst(factorAst);
1853 		} else if (preDec) {
1854 			return new PreDecAst(factorAst);
1855 		} else if (postInc) {
1856 			return new PostIncAst(factorAst);
1857 		} else if (postDec) {
1858 			return new PostDecAst(factorAst);
1859 		} else {
1860 			return factorAst;
1861 		}
1862 	}
1863 
1864 	// FACTOR : '(' ASSIGNMENT_EXPRESSION ')' | Token.INTEGER | Token.DOUBLE | Token.STRING | GETLINE
1865 	// [Token.ID-or-array-or-$val] | /[=].../
1866 	// | [++|--] SYMBOL [++|--]
1867 	// AST FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allow_post_incdec_operators)
1868 	AST FACTOR(boolean allowComparison, boolean allowInKeyword, boolean allowMultidimIndices) throws IOException {
1869 		if (token == Token.OPEN_PAREN) {
1870 			lexer();
1871 			// true = allow multi-dimensional array indices (i.e., commas for 1,2,3,4)
1872 			AST assignmentExpression = ASSIGNMENT_EXPRESSION(null, true, allowInKeyword, true);
1873 			lexer(Token.CLOSE_PAREN);
1874 			if (assignmentExpression instanceof ArrayIndexAst && !(allowInKeyword && token == Token.KW_IN)) {
1875 				// (expr, expr, ...) is a grouping, not an expression: it is only valid
1876 				// immediately before "in", as in ((i, j) in array). This also rejects
1877 				// nested groupings such as (1, (2, 3)), whose inner group is followed
1878 				// by ')' rather than "in".
1879 				throw parserException("A parenthesized expression list is only valid before 'in'.");
1880 			}
1881 			return assignmentExpression;
1882 		} else if (token == Token.INTEGER) {
1883 			AST integer = symbolTable.addINTEGER(text.toString());
1884 			lexer();
1885 			return integer;
1886 		} else if (token == Token.DOUBLE) {
1887 			AST dbl = symbolTable.addDOUBLE(text.toString());
1888 			lexer();
1889 			return dbl;
1890 		} else if (token == Token.STRING) {
1891 			AST str = symbolTable.addSTRING(string.toString());
1892 			lexer();
1893 			return str;
1894 		} else if (token == Token.INDIRECT) {
1895 			return INDIRECT_FUNCTION_CALL(allowInKeyword);
1896 		} else if (token == Token.TYPED_REGEXP) {
1897 			AST regexpAst = symbolTable.addTYPED_REGEXP(regexp.toString());
1898 			lexer();
1899 			return regexpAst;
1900 		} else if (token == Token.KW_GETLINE) {
1901 			return GETLINE_EXPRESSION(null, allowComparison, allowInKeyword);
1902 		} else if (token == Token.DIVIDE || token == Token.DIV_EQ) {
1903 			readRegexp();
1904 			if (token == Token.DIV_EQ) {
1905 				regexp.insert(0, '=');
1906 			}
1907 			AST regexpAst = symbolTable.addREGEXP(regexp.toString());
1908 			lexer();
1909 			return regexpAst;
1910 		} else {
1911 			if (token == Token.DOLLAR) {
1912 				lexer();
1913 				if (token == Token.INC || token == Token.DEC) {
1914 					return new DollarExpressionAst(FACTOR_FOR_INCDEC(allowComparison, allowInKeyword, allowMultidimIndices));
1915 				}
1916 				if (token == Token.NOT || token == Token.MINUS || token == Token.PLUS) {
1917 					return new DollarExpressionAst(UNARY_FACTOR(allowComparison, allowInKeyword, allowMultidimIndices));
1918 				}
1919 				return new DollarExpressionAst(FACTOR(allowComparison, allowInKeyword, allowMultidimIndices));
1920 			}
1921 			return SYMBOL(allowComparison, allowInKeyword);
1922 		}
1923 	}
1924 
1925 	private AST INDIRECT_FUNCTION_CALL(boolean allowInKeyword) throws IOException {
1926 		lexer();
1927 		if (token != Token.ID) {
1928 			throw parserException("An indirect function call requires a variable name after @.");
1929 		}
1930 		AST functionNameAst = symbolTable.getID(text.toString());
1931 		lexer();
1932 		lexer(Token.OPEN_PAREN);
1933 		AST params = token == Token.CLOSE_PAREN ? null : EXPRESSION_LIST(true, allowInKeyword);
1934 		lexer(Token.CLOSE_PAREN);
1935 		return new IndirectFunctionCallAst(functionNameAst, params);
1936 	}
1937 
1938 	// SYMBOL : Token.ID [ '(' params ')' | '[' ASSIGNMENT_EXPRESSION ']' ]
1939 	AST SYMBOL(boolean allowComparison, boolean allowInKeyword) throws IOException {
1940 		if (token != Token.ID && token != Token.FUNC_ID && token != Token.BUILTIN_FUNC_NAME && token != Token.EXTENSION) {
1941 			throw parserException("Expecting an Token.ID. Got " + token.name() + ": " + text);
1942 		}
1943 		Token idToken = token;
1944 		String id = text.toString();
1945 		lexer();
1946 
1947 		if (idToken == Token.EXTENSION) {
1948 			String extensionKeyword = id;
1949 			ExtensionFunction function = extensions.get(extensionKeyword);
1950 			if (function == null) {
1951 				throw parserException("Unknown extension keyword: " + extensionKeyword);
1952 			}
1953 			AST params;
1954 
1955 			/*
1956 			 * if (extension.requiresParen()) {
1957 			 * lexer(Token.OPEN_PAREN);
1958 			 * if (token == Token.CLOSE_PAREN)
1959 			 * params = null;
1960 			 * else
1961 			 * params = EXPRESSION_LIST(allowComparison, allowInKeyword);
1962 			 * lexer(Token.CLOSE_PAREN);
1963 			 * } else {
1964 			 * boolean parens = c == '(';
1965 			 * //expectKeyword("delete");
1966 			 * if (parens) {
1967 			 * assert token == Token.OPEN_PAREN;
1968 			 * lexer();
1969 			 * }
1970 			 * //AST symbolAst = SYMBOL(true,true); // allow comparators
1971 			 * params = EXPRESSION_LIST(allowComparison, allowInKeyword);
1972 			 * if (parens)
1973 			 * lexer(Token.CLOSE_PAREN);
1974 			 * }
1975 			 */
1976 
1977 			// like the built-in functions (and gawk's own builtins, which is
1978 			// what these keywords stand in for), an extension call accepts
1979 			// whitespace between the keyword and its argument list
1980 			if (token == Token.OPEN_PAREN) {
1981 				lexer();
1982 				if (token == Token.CLOSE_PAREN) {
1983 					params = null;
1984 				} else { // comparators allowed, allow “in” inside the extension call
1985 					params = EXPRESSION_LIST(true, allowInKeyword);
1986 				}
1987 				lexer(Token.CLOSE_PAREN);
1988 			} else {
1989 				/*
1990 				 * if (token == Token.NEWLINE || token == Token.SEMICOLON || token == Token.CLOSE_BRACE || token ==
1991 				 * Token.CLOSE_PAREN
1992 				 * || (token == Token.GT || token == Token.APPEND || token == Token.PIPE) )
1993 				 * params = null;
1994 				 * else
1995 				 * params = EXPRESSION_LIST(false,true);
1996 				 */
1997 				params = null;
1998 			}
1999 
2000 			return new ExtensionAst(function, params, extensionCallLineNumber(params));
2001 		} else if (idToken == Token.FUNC_ID || idToken == Token.BUILTIN_FUNC_NAME) {
2002 			AST params;
2003 			// length can take on the special form of no parens
2004 			if (id.equals("length")) {
2005 				if (token == Token.OPEN_PAREN) {
2006 					lexer();
2007 					if (token == Token.CLOSE_PAREN) {
2008 						params = null;
2009 					} else {
2010 						params = EXPRESSION_LIST(true, allowInKeyword);
2011 					}
2012 					lexer(Token.CLOSE_PAREN);
2013 				} else {
2014 					params = null;
2015 				}
2016 			} else {
2017 				lexer(Token.OPEN_PAREN);
2018 				if (token == Token.CLOSE_PAREN) {
2019 					params = null;
2020 				} else {
2021 					params = EXPRESSION_LIST(true, allowInKeyword);
2022 				}
2023 				lexer(Token.CLOSE_PAREN);
2024 			}
2025 			if (idToken == Token.BUILTIN_FUNC_NAME) {
2026 				return new BuiltinFunctionCallAst(id, params);
2027 			} else {
2028 				return symbolTable.addFunctionCall(id, params);
2029 			}
2030 		}
2031 		if (token == Token.OPEN_BRACKET) {
2032 			int arrayReferenceLineNo = currentSourceLineNumber();
2033 			lexer();
2034 			AST idxAst = ARRAY_INDEX(true, allowInKeyword);
2035 			lexer(Token.CLOSE_BRACKET);
2036 			AST arrayReference = symbolTable.addArrayReference(id, idxAst, arrayReferenceLineNo);
2037 			if (posix && token == Token.OPEN_BRACKET) {
2038 				throw parserException("Use [a,b,c,...] instead of [a][b][c]... for multi-dimensional arrays.");
2039 			}
2040 			while (!posix && token == Token.OPEN_BRACKET) {
2041 				int nestedArrayReferenceLineNo = currentSourceLineNumber();
2042 				lexer();
2043 				idxAst = ARRAY_INDEX(true, allowInKeyword);
2044 				lexer(Token.CLOSE_BRACKET);
2045 				arrayReference = new ArrayReferenceAst(nestedArrayReferenceLineNo, arrayReference, idxAst);
2046 			}
2047 			return arrayReference;
2048 		}
2049 		return symbolTable.addID(id);
2050 	}
2051 
2052 	// ARRAY_INDEX : ASSIGNMENT_EXPRESSION [, ARRAY_INDEX]
2053 	AST ARRAY_INDEX(boolean allowComparison, boolean allowInKeyword) throws IOException {
2054 		// Capture the line before parsing the expression: the expression AST can
2055 		// be a shared identifier node carrying its first occurrence's line, and
2056 		// by the end of the expression the lexer may have read past the line.
2057 		int lineNo = currentSourceLineNumber();
2058 		AST exprAst = ASSIGNMENT_EXPRESSION(null, allowComparison, allowInKeyword, false);
2059 		if (token == Token.COMMA) {
2060 			optNewline();
2061 			lexer();
2062 			return new ArrayIndexAst(lineNo, exprAst, ARRAY_INDEX(allowComparison, allowInKeyword));
2063 		} else {
2064 			return new ArrayIndexAst(lineNo, exprAst, null);
2065 		}
2066 	}
2067 
2068 	// STATEMENT :
2069 	// IF_STATEMENT
2070 	// | WHILE_STATEMENT
2071 	// | FOR_STATEMENT
2072 	// | DO_STATEMENT
2073 	// | RETURN_STATEMENT
2074 	// | ASSIGNMENT_EXPRESSION
2075 	AST STATEMENT() throws IOException {
2076 		if (token == Token.OPEN_BRACE) {
2077 			lexer();
2078 			AST lst = STATEMENT_LIST();
2079 			lexer(Token.CLOSE_BRACE);
2080 			return lst;
2081 		}
2082 		AST stmt;
2083 		if (token == Token.KW_IF) {
2084 			stmt = IF_STATEMENT();
2085 		} else if (token == Token.KW_WHILE) {
2086 			stmt = WHILE_STATEMENT();
2087 		} else if (token == Token.KW_FOR) {
2088 			stmt = FOR_STATEMENT();
2089 		} else {
2090 			if (token == Token.KW_DO) {
2091 				stmt = DO_STATEMENT();
2092 			} else if (token == Token.KW_RETURN) {
2093 				stmt = RETURN_STATEMENT();
2094 			} else if (token == Token.KW_EXIT) {
2095 				stmt = EXIT_STATEMENT();
2096 			} else if (token == Token.KW_DELETE) {
2097 				stmt = DELETE_STATEMENT();
2098 			} else if (token == Token.KW_PRINT) {
2099 				stmt = PRINT_STATEMENT();
2100 			} else if (token == Token.KW_PRINTF) {
2101 				stmt = PRINTF_STATEMENT();
2102 			} else if (token == Token.KW_NEXT) {
2103 				stmt = NEXT_STATEMENT();
2104 			} else if (token == Token.KW_NEXTFILE) {
2105 				stmt = NEXTFILE_STATEMENT();
2106 			} else if (token == Token.KW_CONTINUE) {
2107 				stmt = CONTINUE_STATEMENT();
2108 			} else if (token == Token.KW_BREAK) {
2109 				stmt = BREAK_STATEMENT();
2110 			} else {
2111 				stmt = EXPRESSION_STATEMENT(true);
2112 			}
2113 			terminator();
2114 			return stmt;
2115 		}
2116 		// NO TERMINATOR FOR IF, WHILE, Token.AND FOR
2117 		// (leave it for absorption by the callee)
2118 		return stmt;
2119 	}
2120 
2121 	AST EXPRESSION_STATEMENT(boolean allowInKeyword) throws IOException {
2122 		// true = allow comparators
2123 		// false = do Token.NOT allow multi-dimensional array indices
2124 		// return new ExpressionStatementAst(ASSIGNMENT_EXPRESSION(true, allowInKeyword, false));
2125 
2126 		AST exprAst = ASSIGNMENT_EXPRESSION(null, true, allowInKeyword, false);
2127 		return new ExpressionStatementAst(exprAst);
2128 	}
2129 
2130 	AST IF_STATEMENT() throws IOException {
2131 		expectKeyword("if");
2132 		lexer(Token.OPEN_PAREN);
2133 		AST expr = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparators, allow in keyword, do Token.NOT
2134 																																// allow
2135 		// multidim
2136 		// indices expressions
2137 		lexer(Token.CLOSE_PAREN);
2138 
2139 		//// Was:
2140 		//// AST b1 = BLOCK_OR_STMT();
2141 		//// But it didn't handle
2142 		//// if ; else ...
2143 		//// properly
2144 		optNewline();
2145 		AST b1;
2146 		if (token == Token.SEMICOLON) {
2147 			lexer();
2148 			// consume the newline after the semicolon
2149 			optNewline();
2150 			b1 = null;
2151 		} else {
2152 			b1 = BLOCK_OR_STMT();
2153 		}
2154 
2155 		// The OPT_NEWLINE() above causes issues with the following form:
2156 		// if (...) {
2157 		// }
2158 		// else { ... }
2159 		// The \n before the else disassociates subsequent statements
2160 		// if an "else" does not immediately follow.
2161 		// To accommodate, the ifStatement will continue to manage
2162 		// statements, causing the original OPT_STATEMENT_LIST to relinquish
2163 		// processing statements to this OPT_STATEMENT_LIST.
2164 
2165 		optNewline();
2166 		if (token == Token.KW_ELSE) {
2167 			lexer();
2168 			optNewline();
2169 			AST b2 = BLOCK_OR_STMT();
2170 			return new IfStatementAst(expr, b1, b2);
2171 		} else {
2172 			AST ifAst = new IfStatementAst(expr, b1, null);
2173 			return ifAst;
2174 		}
2175 	}
2176 
2177 	AST BREAK_STATEMENT() throws IOException {
2178 		expectKeyword("break");
2179 		return new BreakStatementAst();
2180 	}
2181 
2182 	AST BLOCK_OR_STMT() throws IOException {
2183 		// default case, does Token.NOT consume (require) a terminator
2184 		return BLOCK_OR_STMT(false);
2185 	}
2186 
2187 	AST BLOCK_OR_STMT(boolean requireTerminator) throws IOException {
2188 		optNewline();
2189 		AST block;
2190 		// HIJACK BRACES HERE SINCE WE MAY Token.NOT HAVE A TERMINATOR AFTER THE CLOSING BRACE
2191 		if (token == Token.OPEN_BRACE) {
2192 			lexer();
2193 			block = STATEMENT_LIST();
2194 			lexer(Token.CLOSE_BRACE);
2195 			return block;
2196 		} else if (token == Token.SEMICOLON) {
2197 			block = null;
2198 		} else {
2199 			block = STATEMENT();
2200 			// NO TERMINATOR HERE!
2201 		}
2202 		if (requireTerminator) {
2203 			terminator();
2204 		}
2205 		return block;
2206 	}
2207 
2208 	AST WHILE_STATEMENT() throws IOException {
2209 		expectKeyword("while");
2210 		lexer(Token.OPEN_PAREN);
2211 		AST expr = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparators, allow IN keyword, do Token.NOT
2212 																																// allow
2213 		// multidim
2214 		// indices expressions
2215 		lexer(Token.CLOSE_PAREN);
2216 		AST block = BLOCK_OR_STMT();
2217 		return new WhileStatementAst(expr, block);
2218 	}
2219 
2220 	AST FOR_STATEMENT() throws IOException {
2221 		expectKeyword("for");
2222 		AST expr1 = null;
2223 		AST expr2 = null;
2224 		AST expr3 = null;
2225 		lexer(Token.OPEN_PAREN);
2226 		expr1 = OPT_SIMPLE_STATEMENT(false); // false = "no in keyword allowed"
2227 
2228 		// branch here if we expect a for(... in ...) statement
2229 		if (token == Token.KW_IN) {
2230 			if (expr1.ast1 == null || expr1.ast2 != null) {
2231 				throw parserException("Invalid expression prior to 'in' statement. Got : " + expr1);
2232 			}
2233 			expr1 = expr1.ast1;
2234 			// analyze expr1 to make sure it's a singleton IDAst
2235 			if (!(expr1 instanceof IDAst)) {
2236 				throw parserException("Expecting an Token.ID for 'in' statement. Got : " + expr1);
2237 			}
2238 			// in
2239 			lexer();
2240 			if (token != Token.ID) {
2241 				throw parserException("Expecting an array or subarray for 'in' statement. Got " + token.name() + ": " + text);
2242 			}
2243 			AST arrayAst = SYMBOL(true, true);
2244 			// close paren ...
2245 			lexer(Token.CLOSE_PAREN);
2246 			AST block = BLOCK_OR_STMT();
2247 			return new ForInStatementAst(expr1, arrayAst, block);
2248 		}
2249 
2250 		if (token == Token.SEMICOLON) {
2251 			lexer();
2252 			optNewline();
2253 		} else {
2254 			throw parserException("Expecting ;. Got " + token.name() + ": " + text);
2255 		}
2256 		if (token != Token.SEMICOLON) {
2257 			expr2 = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparators, allow IN keyword, do Token.NOT allow
2258 			// multidim
2259 			// indices expressions
2260 		}
2261 		if (token == Token.SEMICOLON) {
2262 			lexer();
2263 			optNewline();
2264 		} else {
2265 			throw parserException("Expecting ;. Got " + token.name() + ": " + text);
2266 		}
2267 		if (token != Token.CLOSE_PAREN) {
2268 			expr3 = OPT_SIMPLE_STATEMENT(true); // true = "allow the in keyword"
2269 		}
2270 		lexer(Token.CLOSE_PAREN);
2271 		AST block = BLOCK_OR_STMT();
2272 		return new ForStatementAst(expr1, expr2, expr3, block);
2273 	}
2274 
2275 	AST OPT_SIMPLE_STATEMENT(boolean allowInKeyword) throws IOException {
2276 		if (token == Token.SEMICOLON) {
2277 			return null;
2278 		} else if (token == Token.KW_DELETE) {
2279 			return DELETE_STATEMENT();
2280 		} else if (token == Token.KW_PRINT) {
2281 			return PRINT_STATEMENT();
2282 		} else if (token == Token.KW_PRINTF) {
2283 			return PRINTF_STATEMENT();
2284 		} else {
2285 			// allow non-statement ASTs
2286 			return EXPRESSION_STATEMENT(allowInKeyword);
2287 		}
2288 	}
2289 
2290 	AST DELETE_STATEMENT() throws IOException {
2291 		boolean parens = c == '(';
2292 		expectKeyword("delete");
2293 		if (parens) {
2294 			lexer();
2295 		}
2296 		AST symbolAst = SYMBOL(true, true); // allow comparators
2297 		if (parens) {
2298 			lexer(Token.CLOSE_PAREN);
2299 		}
2300 
2301 		return new DeleteStatementAst(symbolAst);
2302 	}
2303 
2304 	private static final class ParsedPrintStatement {
2305 
2306 		private final AST funcParams;
2307 		private final Token outputToken;
2308 		private final AST outputExpr;
2309 		private final boolean parenthesized;
2310 
2311 		ParsedPrintStatement(AST funcParams, Token outputToken, AST outputExpr, boolean parenthesized) {
2312 			this.funcParams = funcParams;
2313 			this.outputToken = outputToken;
2314 			this.outputExpr = outputExpr;
2315 			this.parenthesized = parenthesized;
2316 		}
2317 
2318 		public AST getFuncParams() {
2319 			return funcParams;
2320 		}
2321 
2322 		public Token getOutputToken() {
2323 			return outputToken;
2324 		}
2325 
2326 		public AST getOutputExpr() {
2327 			return outputExpr;
2328 		}
2329 
2330 		public boolean isParenthesized() {
2331 			return parenthesized;
2332 		}
2333 	}
2334 
2335 	private ParsedPrintStatement parsePrintStatement() throws IOException {
2336 		AST funcParams;
2337 		Token outputToken;
2338 		AST outputExpr;
2339 		boolean parenthesized = false;
2340 
2341 		if (token == Token.OPEN_PAREN) {
2342 			parenthesized = true;
2343 			funcParams = parseParenthesizedPrintArguments();
2344 		} else if (endsPrintArgumentList(token)) {
2345 			funcParams = null;
2346 		} else {
2347 			funcParams = EXPRESSION_LIST(false, true); // no comparisons allowed, but allow “in”
2348 		}
2349 
2350 		if (token == Token.GT || token == Token.APPEND || token == Token.PIPE) {
2351 			outputToken = token;
2352 			lexer();
2353 			outputExpr = ASSIGNMENT_EXPRESSION(null, true, true, false); // allow comparisons, “in”; no multidim indices
2354 		} else {
2355 			outputToken = null;
2356 			outputExpr = null;
2357 		}
2358 
2359 		return new ParsedPrintStatement(funcParams, outputToken, outputExpr, parenthesized);
2360 	}
2361 
2362 	private boolean endsPrintArgumentList(Token candidate) {
2363 		return candidate == Token.NEWLINE
2364 				|| candidate == Token.SEMICOLON
2365 				|| candidate == Token.CLOSE_BRACE
2366 				|| candidate == Token.CLOSE_PAREN
2367 				|| candidate == Token.GT
2368 				|| candidate == Token.APPEND
2369 				|| candidate == Token.PIPE
2370 				|| candidate == Token.EOF;
2371 	}
2372 
2373 	private AST parseParenthesizedPrintArguments() throws IOException {
2374 		lexer(); // consume '('
2375 		if (token == Token.CLOSE_PAREN) {
2376 			lexer(); // consume ')'
2377 			return null;
2378 		}
2379 
2380 		AST params = EXPRESSION_LIST(true, true); // allow comparisons and “in” within parentheses
2381 		lexer(Token.CLOSE_PAREN);
2382 
2383 		if (params instanceof FunctionCallParamListAst) {
2384 			FunctionCallParamListAst paramList = (FunctionCallParamListAst) params;
2385 			boolean singleExpression = paramList.getAst2() == null;
2386 			// A parenthesized group followed by "in" is a membership test whose key is
2387 			// the whole group, e.g.: print (1,2) in a
2388 			boolean membershipKey = !singleExpression && token == Token.KW_IN;
2389 			if ((singleExpression || membershipKey) && !endsPrintArgumentList(token)) {
2390 				AST continuedExpression = ASSIGNMENT_EXPRESSION(
2391 						singleExpression ? paramList.getAst1() : toMultidimIndex(paramList),
2392 						false,
2393 						true,
2394 						false);
2395 				if (token == Token.COMMA) {
2396 					// A single parenthesized expression followed by a comma continues the
2397 					// output expression list, e.g.: print (i==0), (i=="")
2398 					lexer(); // consume ','
2399 					optNewline(); // allow newline after comma (AWK style)
2400 					return new FunctionCallParamListAst(continuedExpression, EXPRESSION_LIST(false, true));
2401 				}
2402 				return new FunctionCallParamListAst(continuedExpression, null);
2403 			}
2404 		}
2405 
2406 		return params;
2407 	}
2408 
2409 	// Converts a print argument list back into the multi-dimensional array index
2410 	// it turned out to be, when the parenthesized group is followed by "in"
2411 	private AST toMultidimIndex(FunctionCallParamListAst list) {
2412 		AST rest = list.getAst2() == null ? null : toMultidimIndex((FunctionCallParamListAst) list.getAst2());
2413 		return new ArrayIndexAst(list.getLineNo(), list.getAst1(), rest);
2414 	}
2415 
2416 	AST PRINT_STATEMENT() throws IOException {
2417 		expectKeyword("print");
2418 		ParsedPrintStatement parsedPrintStatement = parsePrintStatement();
2419 
2420 		AST params = parsedPrintStatement.getFuncParams();
2421 		if (parsedPrintStatement.isParenthesized()
2422 				&& token == Token.QUESTION_MARK
2423 				&& params instanceof FunctionCallParamListAst
2424 				&& ((FunctionCallParamListAst) params).getAst2() == null) {
2425 			AST condExpr = ((FunctionCallParamListAst) params).getAst1();
2426 			lexer();
2427 			AST trueBlock = TERNARY_EXPRESSION(null, true, true, true);
2428 			lexer(Token.COLON);
2429 			AST falseBlock = TERNARY_EXPRESSION(null, true, true, true);
2430 			params = new FunctionCallParamListAst(new TernaryExpressionAst(condExpr, trueBlock, falseBlock), null);
2431 		}
2432 
2433 		return new PrintAst(
2434 				params,
2435 				parsedPrintStatement.getOutputToken(),
2436 				parsedPrintStatement.getOutputExpr(),
2437 				parsedPrintStatement.isParenthesized());
2438 	}
2439 
2440 	AST PRINTF_STATEMENT() throws IOException {
2441 		expectKeyword("printf");
2442 		ParsedPrintStatement parsedPrintStatement = parsePrintStatement();
2443 
2444 		AST params = parsedPrintStatement.getFuncParams();
2445 		if (parsedPrintStatement.isParenthesized()
2446 				&& token == Token.QUESTION_MARK
2447 				&& params instanceof FunctionCallParamListAst
2448 				&& ((FunctionCallParamListAst) params).getAst2() == null) {
2449 			AST condExpr = ((FunctionCallParamListAst) params).getAst1();
2450 			lexer();
2451 			AST trueBlock = TERNARY_EXPRESSION(null, true, true, true);
2452 			lexer(Token.COLON);
2453 			AST falseBlock = TERNARY_EXPRESSION(null, true, true, true);
2454 			params = new FunctionCallParamListAst(new TernaryExpressionAst(condExpr, trueBlock, falseBlock), null);
2455 		}
2456 
2457 		return new PrintfAst(params, parsedPrintStatement.getOutputToken(), parsedPrintStatement.getOutputExpr());
2458 	}
2459 
2460 	AST GETLINE_EXPRESSION(AST pipeExpr, boolean allowComparison, boolean allowInKeyword) throws IOException {
2461 		expectKeyword("getline");
2462 		AST lvalue = LVALUE(allowComparison, allowInKeyword);
2463 		if (token == Token.LT) {
2464 			lexer();
2465 			AST assignmentExpr = ASSIGNMENT_EXPRESSION(null, allowComparison, allowInKeyword, false); // do Token.NOT allow
2466 																																																// multidim
2467 			// indices expressions
2468 			return pipeExpr == null ?
2469 					new GetlineAst(null, lvalue, assignmentExpr) :
2470 					new GetlineAst(pipeExpr, lvalue, assignmentExpr);
2471 		} else {
2472 			return pipeExpr == null ? new GetlineAst(null, lvalue, null) : new GetlineAst(pipeExpr, lvalue, null);
2473 		}
2474 	}
2475 
2476 	AST LVALUE(boolean allowComparison, boolean allowInKeyword) throws IOException {
2477 		// false = do Token.NOT allow multi dimension indices expressions
2478 		if (token == Token.DOLLAR) {
2479 			return FACTOR(allowComparison, allowInKeyword, false);
2480 		}
2481 		if (token == Token.ID) {
2482 			return FACTOR(allowComparison, allowInKeyword, false);
2483 		}
2484 		return null;
2485 	}
2486 
2487 	AST DO_STATEMENT() throws IOException {
2488 		expectKeyword("do");
2489 		optNewline();
2490 		AST block = BLOCK_OR_STMT();
2491 		if (token == Token.SEMICOLON) {
2492 			lexer();
2493 		}
2494 		optNewline();
2495 		expectKeyword("while");
2496 		lexer(Token.OPEN_PAREN);
2497 		AST expr = ASSIGNMENT_EXPRESSION(null, true, true, false); // true = allow comparators, allow IN keyword, do
2498 																																// Token.NOT
2499 		// allow
2500 		// multidim indices expressions
2501 		lexer(Token.CLOSE_PAREN);
2502 		return new DoStatementAst(block, expr);
2503 	}
2504 
2505 	AST RETURN_STATEMENT() throws IOException {
2506 		expectKeyword("return");
2507 		if (token == Token.SEMICOLON || token == Token.NEWLINE || token == Token.CLOSE_BRACE) {
2508 			return new ReturnStatementAst(null);
2509 		} else {
2510 			return new ReturnStatementAst(ASSIGNMENT_EXPRESSION(null, true, true, false)); // true = allow comparators, allow
2511 																																											// IN
2512 			// keyword, do Token.NOT allow multidim
2513 			// indices expressions
2514 		}
2515 	}
2516 
2517 	AST EXIT_STATEMENT() throws IOException {
2518 		expectKeyword("exit");
2519 		if (token == Token.SEMICOLON || token == Token.NEWLINE || token == Token.CLOSE_BRACE) {
2520 			return new ExitStatementAst(null);
2521 		} else {
2522 			return new ExitStatementAst(ASSIGNMENT_EXPRESSION(null, true, true, false)); // true = allow comparators, allow IN
2523 			// keyword, do Token.NOT allow multidim
2524 			// indices
2525 			// expressions
2526 		}
2527 	}
2528 
2529 	AST NEXT_STATEMENT() throws IOException {
2530 		expectKeyword("next");
2531 		return new NextStatementAst();
2532 	}
2533 
2534 	AST NEXTFILE_STATEMENT() throws IOException {
2535 		expectKeyword("nextfile");
2536 		nextfileEncountered = true;
2537 		return new NextfileStatementAst();
2538 	}
2539 
2540 	AST CONTINUE_STATEMENT() throws IOException {
2541 		expectKeyword("continue");
2542 		return new ContinueStatementAst();
2543 	}
2544 
2545 	// CHECKSTYLE.ON MethodName
2546 
2547 	private void expectKeyword(String keyword) throws IOException {
2548 		if (token == KEYWORDS.get(keyword)) {
2549 			lexer();
2550 		} else {
2551 			throw parserException("Expecting " + keyword + ". Got " + token.name() + ": " + text);
2552 		}
2553 	}
2554 
2555 	private void populateArrayOperandTuples(
2556 			AST arrayAst,
2557 			AwkTuples tuples,
2558 			boolean createIfMissing,
2559 			String errorMessage) {
2560 		if (arrayAst instanceof IDAst) {
2561 			IDAst idAst = (IDAst) arrayAst;
2562 			idAst.setArray(true);
2563 			if (isJrtManagedSpecialName(idAst.id)) {
2564 				idAst.populateTuples(tuples);
2565 			} else {
2566 				tuples.dereference(idAst.offset, true, idAst.isGlobal);
2567 			}
2568 			return;
2569 		}
2570 		if (arrayAst instanceof ArrayReferenceAst) {
2571 			if (posix) {
2572 				arrayAst.throwSemanticException(errorMessage);
2573 			}
2574 			((ArrayReferenceAst) arrayAst).populateArrayValueTuples(tuples, createIfMissing);
2575 			return;
2576 		}
2577 		arrayAst.throwSemanticException(errorMessage);
2578 	}
2579 
2580 	private int populateActualParameters(
2581 			AwkTuples tuples,
2582 			FunctionCallParamListAst params,
2583 			Set<Integer> arrayParameterIndexes,
2584 			Set<Integer> rawValueParameterIndexes,
2585 			Set<Integer> literalRegexpIndexes,
2586 			int parameterIndex) {
2587 		if (params == null) {
2588 			return 0;
2589 		}
2590 		if (arrayParameterIndexes.contains(Integer.valueOf(parameterIndex))) {
2591 			populateArrayOperandTuples(
2592 					params.getAst1(),
2593 					tuples,
2594 					true,
2595 					"Parameter position " + (parameterIndex + 1) + " must be an array or subarray.");
2596 		} else if (literalRegexpIndexes.contains(Integer.valueOf(parameterIndex))) {
2597 			populateRawRegexpParameterTuples(params.getAst1(), tuples);
2598 		} else if (rawValueParameterIndexes.contains(Integer.valueOf(parameterIndex))) {
2599 			populateRawValueTuples(params.getAst1(), tuples);
2600 		} else {
2601 			params.getAst1().populateTuples(tuples);
2602 		}
2603 		if (params.getAst2() == null) {
2604 			return 1;
2605 		}
2606 		return 1 + populateActualParameters(
2607 				tuples,
2608 				(FunctionCallParamListAst) params.getAst2(),
2609 				arrayParameterIndexes,
2610 				rawValueParameterIndexes,
2611 				literalRegexpIndexes,
2612 				parameterIndex + 1);
2613 	}
2614 
2615 	private int populateActualParameters(
2616 			AwkTuples tuples,
2617 			FunctionCallParamListAst params,
2618 			int... literalRegexpIndexesParam) {
2619 		Set<Integer> literalRegexpIndexes = new HashSet<Integer>();
2620 		for (int idx : literalRegexpIndexesParam) {
2621 			literalRegexpIndexes.add(Integer.valueOf(idx));
2622 		}
2623 		return populateActualParameters(
2624 				tuples,
2625 				params,
2626 				Collections.<Integer>emptySet(),
2627 				Collections.<Integer>emptySet(),
2628 				literalRegexpIndexes,
2629 				0);
2630 	}
2631 
2632 	private int populateActualParametersUpTo(
2633 			AwkTuples tuples,
2634 			FunctionCallParamListAst params,
2635 			int parameterIndex,
2636 			int maxParameterCount) {
2637 		/*
2638 		 * Gawk accepts extra user-function arguments with a runtime warning: the
2639 		 * callee has no local slots for them, but their expressions are still
2640 		 * evaluated for their side effects. Extra arguments are therefore emitted
2641 		 * followed by a POP, and only the formal parameter prefix is counted.
2642 		 */
2643 		if (params == null) {
2644 			return 0;
2645 		}
2646 		if (parameterIndex >= maxParameterCount) {
2647 			// Raw-value evaluation runs the expression's side effects but merely
2648 			// peeks at bare variables, so an untyped variable passed as an extra
2649 			// argument is not autovivified into an assigned scalar.
2650 			populateRawValueTuples(params.getAst1(), tuples);
2651 			tuples.pop();
2652 			populateActualParametersUpTo(
2653 					tuples,
2654 					(FunctionCallParamListAst) params.getAst2(),
2655 					parameterIndex + 1,
2656 					maxParameterCount);
2657 			return 0;
2658 		}
2659 		AST argument = params.getAst1();
2660 		if (argument instanceof IDAst && !isJrtManagedSpecialName(((IDAst) argument).id)) {
2661 			IDAst idAst = (IDAst) argument;
2662 			tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal);
2663 		} else if (argument instanceof ArrayReferenceAst) {
2664 			((ArrayReferenceAst) argument).populateTargetReferenceTuples(tuples);
2665 			tuples.pushIndirectArrayArgument();
2666 		} else {
2667 			argument.populateTuples(tuples);
2668 		}
2669 		if (params.getAst2() == null) {
2670 			return 1;
2671 		}
2672 		return 1 + populateActualParametersUpTo(
2673 				tuples,
2674 				(FunctionCallParamListAst) params.getAst2(),
2675 				parameterIndex + 1,
2676 				maxParameterCount);
2677 	}
2678 
2679 	private void populateRawValueTuples(AST valueAst, AwkTuples tuples) {
2680 		/*
2681 		 * typeof() and isarray() need to inspect an lvalue's current state. A
2682 		 * normal scalar dereference autoconverts an untyped variable into AWK's
2683 		 * assigned blank scalar, which would erase the distinction gawk exposes,
2684 		 * so scalar variables use a non-assigning peek instead.
2685 		 * Array elements need no special opcode: a normal element read already
2686 		 * returns the raw untyped marker for a missing element, and, as in gawk,
2687 		 * brings that element into existence so a later `in`, delete, or for-in
2688 		 * observes it.
2689 		 */
2690 		if (valueAst instanceof IDAst) {
2691 			IDAst idAst = (IDAst) valueAst;
2692 			if (isJrtManagedSpecialName(idAst.id)) {
2693 				idAst.populateTuples(tuples);
2694 			} else {
2695 				tuples.peekDereference(idAst.offset, idAst.isGlobal);
2696 			}
2697 			return;
2698 		}
2699 		valueAst.populateTuples(tuples);
2700 	}
2701 
2702 	private void populateRawRegexpParameterTuples(AST valueAst, AwkTuples tuples) {
2703 		if (valueAst instanceof RegexpAst) {
2704 			((RegexpAst) valueAst).populateRawRegexpTuples(tuples);
2705 			return;
2706 		}
2707 		valueAst.populateTuples(tuples);
2708 	}
2709 
2710 	private int populateIndirectActualParameters(AwkTuples tuples, FunctionCallParamListAst params) {
2711 		if (params == null) {
2712 			return 0;
2713 		}
2714 		AST argument = params.getAst1();
2715 		if (argument instanceof IDAst && !isJrtManagedSpecialName(((IDAst) argument).id)) {
2716 			IDAst idAst = (IDAst) argument;
2717 			tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal);
2718 		} else if (argument instanceof ArrayReferenceAst) {
2719 			((ArrayReferenceAst) argument).populateTargetReferenceTuples(tuples);
2720 			tuples.pushIndirectArrayArgument();
2721 		} else {
2722 			argument.populateTuples(tuples);
2723 		}
2724 		return 1 + populateIndirectActualParameters(tuples, (FunctionCallParamListAst) params.getAst2());
2725 	}
2726 
2727 	private Set<Integer> collectArrayParameterIndexes(FunctionDefAst functionDefAst) {
2728 		Set<Integer> arrayIndexes = new HashSet<Integer>();
2729 		FunctionDefParamListAst fPtr = (FunctionDefParamListAst) functionDefAst.getAst1();
2730 		int index = 0;
2731 		while (fPtr != null) {
2732 			IDAst fparam = symbolTable.getFunctionParameterIDAST(functionDefAst.id, fPtr.id);
2733 			if (fparam.isArray()) {
2734 				arrayIndexes.add(Integer.valueOf(index));
2735 			}
2736 			fPtr = (FunctionDefParamListAst) fPtr.getAst1();
2737 			index++;
2738 		}
2739 		return arrayIndexes;
2740 	}
2741 
2742 	// parser
2743 	// ===============================================================================
2744 	// AST class defs
2745 	private abstract class AST extends AstNode {
2746 
2747 		private final String sourceDescription = currentScriptSource.getDescription();
2748 		// PositionTracker consumes these tuple-emitted source lines at runtime, but
2749 		// AST nodes have to capture them here during parsing before tuples exist.
2750 		private final int lineNo;
2751 		private AST parent;
2752 		private AST ast1, ast2, ast3, ast4;
2753 		private final EnumSet<AstFlag> flags = EnumSet.noneOf(AstFlag.class);
2754 
2755 		protected final void addFlag(AstFlag flag) {
2756 			flags.add(flag);
2757 		}
2758 
2759 		protected final boolean hasFlag(AstFlag flag) {
2760 			return flags.contains(flag);
2761 		}
2762 
2763 		protected Address breakAddress() {
2764 			return null;
2765 		}
2766 
2767 		protected Address continueAddress() {
2768 			return null;
2769 		}
2770 
2771 		protected Address nextAddress() {
2772 			return null;
2773 		}
2774 
2775 		protected Address returnAddress() {
2776 			return null;
2777 		}
2778 
2779 		protected final AST getParent() {
2780 			return parent;
2781 		}
2782 
2783 		@SuppressWarnings("unused")
2784 		protected final void setParent(AST p) {
2785 			parent = p;
2786 		}
2787 
2788 		protected final AST getAst1() {
2789 			return ast1;
2790 		}
2791 
2792 		@SuppressWarnings("unused")
2793 		protected final void setAst1(AST a1) {
2794 			ast1 = a1;
2795 		}
2796 
2797 		protected final AST getAst2() {
2798 			return ast2;
2799 		}
2800 
2801 		@SuppressWarnings("unused")
2802 		protected final void setAst2(AST a2) {
2803 			ast2 = a2;
2804 		}
2805 
2806 		protected final AST getAst3() {
2807 			return ast3;
2808 		}
2809 
2810 		@SuppressWarnings("unused")
2811 		protected final void setAst3(AST a3) {
2812 			ast3 = a3;
2813 		}
2814 
2815 		protected final AST getAst4() {
2816 			return ast4;
2817 		}
2818 
2819 		@SuppressWarnings("unused")
2820 		protected final void setAst4(AST a4) {
2821 			ast4 = a4;
2822 		}
2823 
2824 		protected final AST searchFor(AstFlag flag) {
2825 			AST ptr = this;
2826 			while (ptr != null) {
2827 				if (ptr.hasFlag(flag)) {
2828 					return ptr;
2829 				}
2830 				ptr = ptr.parent;
2831 			}
2832 			return null;
2833 		}
2834 
2835 		protected AST() {
2836 			this(currentSourceLineNumber());
2837 		}
2838 
2839 		protected AST(int lineNo) {
2840 			this.lineNo = lineNo;
2841 		}
2842 
2843 		protected int getLineNo() {
2844 			return lineNo;
2845 		}
2846 
2847 		protected String getSourceDescription() {
2848 			return sourceDescription;
2849 		}
2850 
2851 		protected String sourceBasename() {
2852 			// File.getName is a pure string operation and, unlike java.nio.Path,
2853 			// never rejects non-path descriptions such as <command-line-supplied-script>
2854 			return new File(getSourceDescription()).getName();
2855 		}
2856 
2857 		protected AST(AST ast1) {
2858 			this(currentSourceLineNumber(), ast1);
2859 		}
2860 
2861 		protected AST(int lineNo, AST ast1) {
2862 			this(lineNo);
2863 			this.ast1 = ast1;
2864 
2865 			if (ast1 != null) {
2866 				ast1.parent = this;
2867 			}
2868 		}
2869 
2870 		protected AST(AST ast1, AST ast2) {
2871 			this(currentSourceLineNumber(), ast1, ast2);
2872 		}
2873 
2874 		protected AST(int lineNo, AST ast1, AST ast2) {
2875 			this(lineNo);
2876 			this.ast1 = ast1;
2877 			this.ast2 = ast2;
2878 
2879 			if (ast1 != null) {
2880 				ast1.parent = this;
2881 			}
2882 			if (ast2 != null) {
2883 				ast2.parent = this;
2884 			}
2885 		}
2886 
2887 		protected AST(AST ast1, AST ast2, AST ast3) {
2888 			this(currentSourceLineNumber(), ast1, ast2, ast3);
2889 		}
2890 
2891 		protected AST(int lineNo, AST ast1, AST ast2, AST ast3) {
2892 			this(lineNo);
2893 			this.ast1 = ast1;
2894 			this.ast2 = ast2;
2895 			this.ast3 = ast3;
2896 
2897 			if (ast1 != null) {
2898 				ast1.parent = this;
2899 			}
2900 			if (ast2 != null) {
2901 				ast2.parent = this;
2902 			}
2903 			if (ast3 != null) {
2904 				ast3.parent = this;
2905 			}
2906 		}
2907 
2908 		protected AST(AST ast1, AST ast2, AST ast3, AST ast4) {
2909 			this(currentSourceLineNumber(), ast1, ast2, ast3, ast4);
2910 		}
2911 
2912 		protected AST(int lineNo, AST ast1, AST ast2, AST ast3, AST ast4) {
2913 			this(lineNo);
2914 			this.ast1 = ast1;
2915 			this.ast2 = ast2;
2916 			this.ast3 = ast3;
2917 			this.ast4 = ast4;
2918 
2919 			if (ast1 != null) {
2920 				ast1.parent = this;
2921 			}
2922 			if (ast2 != null) {
2923 				ast2.parent = this;
2924 			}
2925 			if (ast3 != null) {
2926 				ast3.parent = this;
2927 			}
2928 			if (ast4 != null) {
2929 				ast4.parent = this;
2930 			}
2931 		}
2932 
2933 		/**
2934 		 * Dump a meaningful text representation of this
2935 		 * abstract syntax tree node to the output (print)
2936 		 * stream. Either it is called directly by the
2937 		 * application program, or it is called by the
2938 		 * parent node of this tree node.
2939 		 *
2940 		 * @param ps The print stream to dump the text
2941 		 *        representation.
2942 		 */
2943 		@Override
2944 		public void dump(PrintStream ps) {
2945 			dump(ps, 0);
2946 		}
2947 
2948 		private void dump(PrintStream ps, int lvl) {
2949 			StringBuffer spaces = new StringBuffer();
2950 			for (int i = 0; i < lvl; i++) {
2951 				spaces.append(' ');
2952 			}
2953 			ps.println(spaces + toString());
2954 			if (ast1 != null) {
2955 				ast1.dump(ps, lvl + 1);
2956 			}
2957 			if (ast2 != null) {
2958 				ast2.dump(ps, lvl + 1);
2959 			}
2960 			if (ast3 != null) {
2961 				ast3.dump(ps, lvl + 1);
2962 			}
2963 			if (ast4 != null) {
2964 				ast4.dump(ps, lvl + 1);
2965 			}
2966 		}
2967 
2968 		/**
2969 		 * Apply semantic checks to this node. The default
2970 		 * implementation is to simply call semanticAnalysis()
2971 		 * on all the children of this abstract syntax tree node.
2972 		 * Therefore, this method must be overridden to provide
2973 		 * meaningful semantic analysis / checks.
2974 		 *
2975 		 * @throws SemanticException upon a semantic error.
2976 		 */
2977 		@Override
2978 		public void semanticAnalysis() {
2979 			if (ast1 != null) {
2980 				ast1.semanticAnalysis();
2981 			}
2982 			if (ast2 != null) {
2983 				ast2.semanticAnalysis();
2984 			}
2985 			if (ast3 != null) {
2986 				ast3.semanticAnalysis();
2987 			}
2988 			if (ast4 != null) {
2989 				ast4.semanticAnalysis();
2990 			}
2991 		}
2992 
2993 		/**
2994 		 * Appends tuples to the AwkTuples list
2995 		 * for this abstract syntax tree node. Subclasses
2996 		 * must implement this method.
2997 		 * <p>
2998 		 * This is called either by the main program to generate a full
2999 		 * list of tuples for the abstract syntax tree, or it is called
3000 		 * by other abstract syntax tree nodes in response to their
3001 		 * attempt at populating tuples.
3002 		 *
3003 		 * @param tuples The tuples to populate.
3004 		 * @return The number of items left on the stack after
3005 		 *         these tuples have executed.
3006 		 */
3007 		@Override
3008 		public abstract int populateTuples(AwkTuples tuples);
3009 
3010 		protected final void pushSourceLineNumber(AwkTuples tuples) {
3011 			tuples.pushSourceLineNumber(lineNo);
3012 		}
3013 
3014 		protected final void popSourceLineNumber(AwkTuples tuples) {
3015 			tuples.popSourceLineNumber(lineNo);
3016 		}
3017 
3018 		private boolean isBegin = isBegin();
3019 
3020 		@SuppressWarnings("unused")
3021 		protected final boolean isBeginFlag() {
3022 			return isBegin;
3023 		}
3024 
3025 		protected final void setBeginFlag(boolean flag) {
3026 			isBegin = flag;
3027 		}
3028 
3029 		private boolean isBegin() {
3030 			boolean result = isBegin;
3031 			if (!result && ast1 != null) {
3032 				result = ast1.isBegin();
3033 			}
3034 			if (!result && ast2 != null) {
3035 				result = ast2.isBegin();
3036 			}
3037 			if (!result && ast3 != null) {
3038 				result = ast3.isBegin();
3039 			}
3040 			if (!result && ast4 != null) {
3041 				result = ast4.isBegin();
3042 			}
3043 			return result;
3044 		}
3045 
3046 		private boolean isEnd = isEnd();
3047 
3048 		@SuppressWarnings("unused")
3049 		protected final boolean isEndFlag() {
3050 			return isEnd;
3051 		}
3052 
3053 		protected final void setEndFlag(boolean flag) {
3054 			isEnd = flag;
3055 		}
3056 
3057 		private boolean isEnd() {
3058 			boolean result = isEnd;
3059 			if (!result && ast1 != null) {
3060 				result = ast1.isEnd();
3061 			}
3062 			if (!result && ast2 != null) {
3063 				result = ast2.isEnd();
3064 			}
3065 			if (!result && ast3 != null) {
3066 				result = ast3.isEnd();
3067 			}
3068 			if (!result && getAst4() != null) {
3069 				result = getAst4().isEnd();
3070 			}
3071 			return result;
3072 		}
3073 
3074 		private boolean isBeginFile = isBeginFile();
3075 
3076 		protected final void setBeginFileFlag(boolean flag) {
3077 			isBeginFile = flag;
3078 		}
3079 
3080 		private boolean isBeginFile() {
3081 			boolean result = isBeginFile;
3082 			if (!result && ast1 != null) {
3083 				result = ast1.isBeginFile();
3084 			}
3085 			if (!result && ast2 != null) {
3086 				result = ast2.isBeginFile();
3087 			}
3088 			if (!result && ast3 != null) {
3089 				result = ast3.isBeginFile();
3090 			}
3091 			if (!result && ast4 != null) {
3092 				result = ast4.isBeginFile();
3093 			}
3094 			return result;
3095 		}
3096 
3097 		private boolean isEndFile = isEndFile();
3098 
3099 		protected final void setEndFileFlag(boolean flag) {
3100 			isEndFile = flag;
3101 		}
3102 
3103 		private boolean isEndFile() {
3104 			boolean result = isEndFile;
3105 			if (!result && ast1 != null) {
3106 				result = ast1.isEndFile();
3107 			}
3108 			if (!result && ast2 != null) {
3109 				result = ast2.isEndFile();
3110 			}
3111 			if (!result && ast3 != null) {
3112 				result = ast3.isEndFile();
3113 			}
3114 			if (!result && ast4 != null) {
3115 				result = ast4.isEndFile();
3116 			}
3117 			return result;
3118 		}
3119 
3120 		private boolean isFunction = isFunction();
3121 
3122 		@SuppressWarnings("unused")
3123 		protected final boolean isFunctionFlag() {
3124 			return isFunction;
3125 		}
3126 
3127 		protected final void setFunctionFlag(boolean flag) {
3128 			isFunction = flag;
3129 		}
3130 
3131 		private boolean isFunction() {
3132 			boolean result = isFunction;
3133 			if (!result && getAst1() != null) {
3134 				result = getAst1().isFunction();
3135 			}
3136 			if (!result && getAst2() != null) {
3137 				result = getAst2().isFunction();
3138 			}
3139 			if (!result && getAst3() != null) {
3140 				result = getAst3().isFunction();
3141 			}
3142 			if (!result && getAst4() != null) {
3143 				result = getAst4().isFunction();
3144 			}
3145 			return result;
3146 		}
3147 
3148 		public boolean isArray() {
3149 			return false;
3150 		}
3151 
3152 		public boolean isScalar() {
3153 			return false;
3154 		}
3155 
3156 		/**
3157 		 * Made protected so that subclasses can access it.
3158 		 * Package-level access was not necessary.
3159 		 */
3160 		protected class SemanticException extends RuntimeException {
3161 
3162 			private static final long serialVersionUID = 1L;
3163 
3164 			SemanticException(String msg) {
3165 				super(msg + " (" + sourceDescription + ":" + lineNo + ")");
3166 			}
3167 		}
3168 
3169 		protected final void throwSemanticException(String msg) {
3170 			throw new SemanticException(msg);
3171 		}
3172 
3173 		@Override
3174 		public String toString() {
3175 			return getClass().getName().replaceFirst(".*[$.]", "");
3176 		}
3177 	}
3178 
3179 	private abstract class ScalarExpressionAst extends AST {
3180 
3181 		protected ScalarExpressionAst() {
3182 			super();
3183 		}
3184 
3185 		protected ScalarExpressionAst(int lineNo) {
3186 			super(lineNo);
3187 		}
3188 
3189 		protected ScalarExpressionAst(AST a1) {
3190 			super(a1);
3191 		}
3192 
3193 		protected ScalarExpressionAst(int lineNo, AST a1) {
3194 			super(lineNo, a1);
3195 		}
3196 
3197 		protected ScalarExpressionAst(AST a1, AST a2) {
3198 			super(a1, a2);
3199 		}
3200 
3201 		protected ScalarExpressionAst(int lineNo, AST a1, AST a2) {
3202 			super(lineNo, a1, a2);
3203 		}
3204 
3205 		protected ScalarExpressionAst(AST a1, AST a2, AST a3) {
3206 			super(a1, a2, a3);
3207 		}
3208 
3209 		protected ScalarExpressionAst(int lineNo, AST a1, AST a2, AST a3) {
3210 			super(lineNo, a1, a2, a3);
3211 		}
3212 
3213 		@Override
3214 		public boolean isArray() {
3215 			return false;
3216 		}
3217 
3218 		@Override
3219 		public boolean isScalar() {
3220 			return true;
3221 		}
3222 	}
3223 
3224 	private static boolean isRule(AST ast) {
3225 		return ast != null && !ast.isBegin() && !ast.isEnd() && !ast.isBeginFile() && !ast.isEndFile() && !ast.isFunction();
3226 	}
3227 
3228 	/**
3229 	 * Inspects the action rule condition whether it contains
3230 	 * extensions. It does a superficial check of
3231 	 * the abstract syntax tree of the action rule.
3232 	 * In other words, it will not examine whether user-defined
3233 	 * functions within the action rule contain extensions.
3234 	 *
3235 	 * @param ast The action rule expression to examine.
3236 	 * @return true if the action rule condition contains
3237 	 *         an extension; false otherwise.
3238 	 */
3239 	@SuppressWarnings("unused")
3240 	private static boolean isExtensionConditionRule(AST ast) {
3241 		if (!isRule(ast)) {
3242 			return false;
3243 		}
3244 		if (ast.getAst1() == null) {
3245 			return false;
3246 		}
3247 
3248 		if (!containsASTType(ast.getAst1(), ExtensionAst.class)) {
3249 			return false;
3250 		}
3251 
3252 		if (containsASTType(ast.getAst1(), new Class[] { FunctionCallAst.class, DollarExpressionAst.class })) {
3253 			return false;
3254 		}
3255 
3256 		return true;
3257 	}
3258 
3259 	private static boolean containsASTType(AST ast, Class<?> cls) {
3260 		return containsASTType(ast, new Class[] { cls });
3261 	}
3262 
3263 	private static boolean containsASTType(AST ast, Class<?>[] clsArray) {
3264 		if (ast == null) {
3265 			return false;
3266 		}
3267 		for (Class<?> cls : clsArray) {
3268 			if (cls.isInstance(ast)) {
3269 				return true;
3270 			}
3271 		}
3272 		// prettier-ignore
3273 		return containsASTType(ast.getAst1(), clsArray)
3274 				|| containsASTType(ast.getAst2(), clsArray)
3275 				|| containsASTType(ast.getAst3(), clsArray)
3276 				|| containsASTType(ast.getAst4(), clsArray);
3277 	}
3278 
3279 	private Address nextAddress;
3280 
3281 	/**
3282 	 * Whether the program contains at least one {@code nextfile} statement,
3283 	 * anywhere (rules or user-defined functions). When set, the main input
3284 	 * loop is compiled with per-file stepping so the runtime can jump to the
3285 	 * ENDFILE section and advance to the next input file.
3286 	 */
3287 	private boolean nextfileEncountered;
3288 
3289 	private final class RuleListAst extends AST {
3290 
3291 		private RuleListAst(AST rule, AST rest) {
3292 			super(rule, rest);
3293 		}
3294 
3295 		@Override
3296 		public int populateTuples(AwkTuples tuples) {
3297 
3298 			pushSourceLineNumber(tuples);
3299 
3300 			nextAddress = tuples.createAddress("nextAddress");
3301 
3302 			// goto start address
3303 			Address startAddress = tuples.createAddress("start address");
3304 			tuples.gotoAddress(startAddress);
3305 
3306 			AST ptr;
3307 
3308 			// compile functions
3309 			ptr = this;
3310 			while (ptr != null) {
3311 				if (ptr.getAst1() != null && ptr.getAst1().isFunction()) {
3312 					ptr.getAst1().populateTuples(tuples);
3313 				}
3314 
3315 				ptr = ptr.getAst2();
3316 			}
3317 
3318 			// START OF MAIN BLOCK
3319 			tuples.address(startAddress);
3320 
3321 			// initialize runtime-managed special variables via JRT defaults in AVM
3322 			symbolTable.getID("NR");
3323 			symbolTable.getID("FNR");
3324 			symbolTable.getID("NF");
3325 			symbolTable.getID("FS");
3326 			symbolTable.getID("RS");
3327 			symbolTable.getID("OFS");
3328 			symbolTable.getID("ORS");
3329 			symbolTable.getID("RSTART");
3330 			symbolTable.getID("RLENGTH");
3331 			symbolTable.getID("FILENAME");
3332 			symbolTable.getID("SUBSEP");
3333 			symbolTable.getID("CONVFMT");
3334 			symbolTable.getID("OFMT");
3335 			IDAst environAst = symbolTable.getID("ENVIRON");
3336 			IDAst argcAst = symbolTable.getID("ARGC");
3337 			IDAst argvAst = symbolTable.getID("ARGV");
3338 
3339 			// MUST BE DONE AFTER FUNCTIONS ARE COMPILED,
3340 			// and after special variables are made known to the symbol table
3341 			// (see above)!
3342 			tuples.setNumGlobals(symbolTable.numGlobals());
3343 
3344 			// Only ENVIRON/ARGC/ARGV remain regular globals. ENVIRON and ARGV
3345 			// are materialized only when the script references them, or when
3346 			// SYMTAB is active (its snapshot exposes them); unreferenced ones
3347 			// are answered by the synthetic accessors. ARGC is always
3348 			// materialized (a single cheap assignment): its slot must stay
3349 			// authoritative so ARGC=n command-line operand assignments affect
3350 			// input traversal.
3351 			boolean symtabActive = !posix && symbolTable.isGlobalReferenced("SYMTAB");
3352 			if (environAst.isReferenced() || symtabActive) {
3353 				tuples.environOffset(environAst.offset);
3354 			}
3355 			tuples.argcOffset(argcAst.offset);
3356 			if (argvAst.isReferenced() || symtabActive) {
3357 				tuples.argvOffset(argvAst.offset);
3358 			}
3359 			// SYMTAB and FUNCTAB are gawk extensions, not POSIX
3360 			if (symtabActive) {
3361 				tuples.updateSymtab(symbolTable.getID("SYMTAB").offset);
3362 			}
3363 			if (!posix && symbolTable.isGlobalReferenced("FUNCTAB")) {
3364 				tuples.updateFunctab(symbolTable.getID("FUNCTAB").offset);
3365 			}
3366 			tuples.beforeStartHooks();
3367 
3368 			Address exitAddr = tuples.createAddress("end blocks start address");
3369 			tuples.setExitAddress(exitAddr);
3370 
3371 			// Does the program use BEGINFILE/ENDFILE rules or nextfile? If so,
3372 			// the main input loop must step through the input one file at a
3373 			// time (per-file scaffolding) instead of streaming across files.
3374 			boolean hasBeginFileRules = false;
3375 			boolean hasEndFileRules = false;
3376 			ptr = this;
3377 			while (ptr != null) {
3378 				if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) {
3379 					hasBeginFileRules = true;
3380 				}
3381 				if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) {
3382 					hasEndFileRules = true;
3383 				}
3384 				ptr = ptr.getAst2();
3385 			}
3386 
3387 			// Do we have rules? (apart from BEGIN)
3388 			// If we have rules, END, BEGINFILE, or ENDFILE, we need to parse
3389 			// the input
3390 			boolean reqInput = hasBeginFileRules || hasEndFileRules;
3391 
3392 			// Check for "normal" rules
3393 			ptr = this;
3394 			while (!reqInput && (ptr != null)) {
3395 				if (isRule(ptr.getAst1())) {
3396 					reqInput = true;
3397 				}
3398 				ptr = ptr.getAst2();
3399 			}
3400 
3401 			// Now check for "END" rules
3402 			ptr = this;
3403 			while (!reqInput && (ptr != null)) {
3404 				if (ptr.getAst1() != null && ptr.getAst1().isEnd()) {
3405 					reqInput = true;
3406 				}
3407 				ptr = ptr.getAst2();
3408 			}
3409 
3410 			boolean perFileScaffolding = reqInput && (hasBeginFileRules || hasEndFileRules || nextfileEncountered);
3411 
3412 			// The per-file addresses are carried as properties of the tuple
3413 			// stream, so a nextfile executed from a user-defined function can
3414 			// resolve them at runtime. The begin_file address labels the
3415 			// NEXT_FILE tuple that opens each input file.
3416 			Address beginFileAddress = null;
3417 			Address endFileAddress = null;
3418 			if (perFileScaffolding) {
3419 				beginFileAddress = tuples.createAddress("begin_file");
3420 				endFileAddress = tuples.createAddress("end_file");
3421 				// The ENDFILE address is registered only when BEGINFILE or
3422 				// ENDFILE rules exist: its presence also confines a
3423 				// non-redirected getline to the current input file, which
3424 				// only matters when there are per-file hooks to protect.
3425 				if (hasBeginFileRules || hasEndFileRules) {
3426 					tuples.setEndFileAddress(endFileAddress);
3427 				}
3428 				tuples.setNextFileAddress(beginFileAddress);
3429 			}
3430 
3431 			// grab all BEGINs
3432 			ptr = this;
3433 			// ptr.getAst1() == blank rule condition (i.e.: { print })
3434 			while (ptr != null) {
3435 				if (ptr.getAst1() != null && ptr.getAst1().isBegin()) {
3436 					ptr.getAst1().populateTuples(tuples);
3437 				}
3438 
3439 				ptr = ptr.getAst2();
3440 			}
3441 
3442 			if (reqInput) {
3443 				Address inputLoopAddress = tuples.createAddress("input_loop_address");
3444 				Address noMoreInput = tuples.createAddress("no_more_input");
3445 
3446 				if (perFileScaffolding) {
3447 					// Advance to the next input file so the BEGINFILE rules
3448 					// observe its FILENAME, FNR, ARGIND, and ERRNO. The
3449 					// ENDFILE section loops back here for the following files.
3450 					tuples.address(beginFileAddress);
3451 					tuples.nextFile(noMoreInput);
3452 
3453 					// BEGINFILE rules, in the order they were read
3454 					ptr = this;
3455 					while (ptr != null) {
3456 						if (ptr.getAst1() != null && ptr.getAst1().isBeginFile()) {
3457 							ptr.getAst1().populateTuples(tuples);
3458 						}
3459 						ptr = ptr.getAst2();
3460 					}
3461 
3462 					// per-file input loop: at end of the current file, run the
3463 					// ENDFILE rules instead of silently opening the next file
3464 					tuples.address(inputLoopAddress);
3465 					tuples.consumeFileInput(endFileAddress);
3466 				} else {
3467 					tuples.address(inputLoopAddress);
3468 					tuples.consumeInput(noMoreInput);
3469 				}
3470 
3471 				// grab all INPUT RULES
3472 				ptr = this;
3473 				while (ptr != null) {
3474 					// the first one of these is an input rule
3475 					if (isRule(ptr.getAst1())) {
3476 						ptr.getAst1().populateTuples(tuples);
3477 					}
3478 					ptr = ptr.getAst2();
3479 				}
3480 				// The main-loop resume point of a runtime next executed from
3481 				// a user-defined function, carried as a property of the tuple
3482 				// stream like the per-file addresses.
3483 				tuples.setNextAddress(nextAddress);
3484 				tuples.address(nextAddress);
3485 
3486 				tuples.gotoAddress(inputLoopAddress);
3487 
3488 				if (perFileScaffolding) {
3489 					// ENDFILE rules, in the order they were read
3490 					tuples.address(endFileAddress);
3491 					ptr = this;
3492 					while (ptr != null) {
3493 						if (ptr.getAst1() != null && ptr.getAst1().isEndFile()) {
3494 							ptr.getAst1().populateTuples(tuples);
3495 						}
3496 						ptr = ptr.getAst2();
3497 					}
3498 
3499 					// then loop back to the NEXT_FILE tuple at begin_file,
3500 					// which falls through to the END rules once the input is
3501 					// exhausted
3502 					tuples.gotoAddress(beginFileAddress);
3503 				}
3504 
3505 				tuples.address(noMoreInput);
3506 				// compiler has issue with missing nop here
3507 				tuples.nop();
3508 			}
3509 
3510 			// indicate where the first end block resides
3511 			// in the event of an exit statement
3512 			tuples.address(exitAddr);
3513 			tuples.setWithinEndBlocks(true);
3514 
3515 			// grab all ENDs
3516 			ptr = this;
3517 			while (ptr != null) {
3518 				if (ptr.getAst1() != null && ptr.getAst1().isEnd()) {
3519 					ptr.getAst1().populateTuples(tuples);
3520 				}
3521 				ptr = ptr.getAst2();
3522 			}
3523 
3524 			// force a nop here to resolve any addresses that haven't been resolved yet
3525 			// (i.e., no_more_input wouldn't be resolved if there are no END{} blocks)
3526 			tuples.nop();
3527 
3528 			popSourceLineNumber(tuples);
3529 			return 0;
3530 		}
3531 	}
3532 
3533 	private final class ExpressionToEvaluateAst extends AST {
3534 
3535 		private ExpressionToEvaluateAst(AST expr) {
3536 			super(expr);
3537 		}
3538 
3539 		@Override
3540 		public int populateTuples(AwkTuples tuples) {
3541 
3542 			pushSourceLineNumber(tuples);
3543 
3544 			// initialize runtime-managed special variables via JRT defaults in AVM
3545 			symbolTable.getID("NR");
3546 			symbolTable.getID("FNR");
3547 			symbolTable.getID("NF");
3548 			symbolTable.getID("FS");
3549 			symbolTable.getID("RS");
3550 			symbolTable.getID("SUBSEP");
3551 			symbolTable.getID("CONVFMT");
3552 			IDAst environAst = symbolTable.getID("ENVIRON");
3553 
3554 			// MUST BE DONE AFTER FUNCTIONS ARE COMPILED,
3555 			// and after special variables are made known to the symbol table
3556 			// (see above)!
3557 			tuples.markEvalTupleStream();
3558 			tuples.setNumGlobals(symbolTable.numGlobals());
3559 
3560 			boolean evalSymtabActive = !posix && symbolTable.isGlobalReferenced("SYMTAB");
3561 			if (environAst.isReferenced() || evalSymtabActive) {
3562 				tuples.environOffset(environAst.offset);
3563 			}
3564 			if (evalSymtabActive) {
3565 				tuples.updateSymtab(symbolTable.getID("SYMTAB").offset);
3566 			}
3567 			if (!posix && symbolTable.isGlobalReferenced("FUNCTAB")) {
3568 				tuples.updateFunctab(symbolTable.getID("FUNCTAB").offset);
3569 			}
3570 			tuples.beforeStartHooks();
3571 
3572 			if (getAst1() != null) {
3573 				getAst1().populateTuples(tuples);
3574 			}
3575 			// Some expression forms (for example ternaries) still need a concrete
3576 			// terminal tuple to resolve branch targets during post-processing.
3577 			tuples.nop();
3578 
3579 			popSourceLineNumber(tuples);
3580 			return 0;
3581 		}
3582 	}
3583 
3584 	// made non-static to access the "nextAddress" field of the frontend
3585 	private final class RuleAst extends AST {
3586 
3587 		private RuleAst(AST optExpression, AST optRule) {
3588 			super(optExpression, optRule);
3589 			addFlag(AstFlag.NEXTABLE);
3590 		}
3591 
3592 		@Override
3593 		public int populateTuples(AwkTuples tuples) {
3594 			pushSourceLineNumber(tuples);
3595 			boolean unconditionalRule = getAst1() == null
3596 					|| getAst1().isBegin()
3597 					|| getAst1().isEnd()
3598 					|| getAst1().isBeginFile()
3599 					|| getAst1().isEndFile();
3600 			if (!unconditionalRule) {
3601 				getAst1().populateTuples(tuples);
3602 				// result of whether to execute or not is on the stack
3603 				Address bypassRule = tuples.createAddress("bypassRule");
3604 				tuples.ifFalse(bypassRule);
3605 				populateRuleBody(tuples);
3606 				tuples.address(bypassRule).nop();
3607 			} else {
3608 				populateRuleBody(tuples);
3609 			}
3610 			popSourceLineNumber(tuples);
3611 			return 0;
3612 		}
3613 
3614 		private void populateRuleBody(AwkTuples tuples) {
3615 			// execute the optRule here!
3616 			if (getAst2() == null) {
3617 				if (isRule(this)) {
3618 					// display $0
3619 					tuples.print(0);
3620 				}
3621 				// else, don't populate it with anything
3622 				// (i.e., blank BEGIN/END/BEGINFILE/ENDFILE rule)
3623 			} else {
3624 				// execute it, and leave nothing on the stack
3625 				getAst2().populateTuples(tuples);
3626 			}
3627 		}
3628 
3629 		@Override
3630 		public Address nextAddress() {
3631 			if (!isRule(this)) {
3632 				throw new SemanticException("`next' cannot be called from a `" + specialRuleName() + "' rule.");
3633 			}
3634 			if (nextAddress == null) {
3635 				throw new SemanticException("Cannot call next here.");
3636 			}
3637 			return nextAddress;
3638 		}
3639 
3640 		/**
3641 		 * Names the special (non-input) rule this AST represents, for
3642 		 * gawk-compatible diagnostics.
3643 		 */
3644 		private String specialRuleName() {
3645 			AST pattern = getAst1();
3646 			if (pattern != null && pattern.isBegin()) {
3647 				return "BEGIN";
3648 			}
3649 			if (pattern != null && pattern.isEnd()) {
3650 				return "END";
3651 			}
3652 			if (pattern != null && pattern.isBeginFile()) {
3653 				return "BEGINFILE";
3654 			}
3655 			if (pattern != null && pattern.isEndFile()) {
3656 				return "ENDFILE";
3657 			}
3658 			return "special";
3659 		}
3660 	}
3661 
3662 	private final class IfStatementAst extends AST {
3663 
3664 		private IfStatementAst(AST expr, AST b1, AST b2) {
3665 			super(expr, b1, b2);
3666 		}
3667 
3668 		@Override
3669 		public int populateTuples(AwkTuples tuples) {
3670 			pushSourceLineNumber(tuples);
3671 
3672 			Address elseblock = tuples.createAddress("elseblock");
3673 
3674 			getAst1().populateTuples(tuples);
3675 			tuples.ifFalse(elseblock);
3676 			if (getAst2() != null) {
3677 				getAst2().populateTuples(tuples);
3678 			}
3679 			if (getAst3() == null) {
3680 				tuples.address(elseblock);
3681 			} else {
3682 				Address end = tuples.createAddress("end");
3683 				tuples.gotoAddress(end);
3684 				tuples.address(elseblock);
3685 				getAst3().populateTuples(tuples);
3686 				tuples.address(end);
3687 			}
3688 			popSourceLineNumber(tuples);
3689 			return 0;
3690 		}
3691 	}
3692 
3693 	private final class TernaryExpressionAst extends ScalarExpressionAst {
3694 
3695 		private TernaryExpressionAst(AST a1, AST a2, AST a3) {
3696 			super(a1, a2, a3);
3697 		}
3698 
3699 		@Override
3700 		public int populateTuples(AwkTuples tuples) {
3701 			pushSourceLineNumber(tuples);
3702 
3703 			Address elseexpr = tuples.createAddress("elseexpr");
3704 			Address endTertiary = tuples.createAddress("endTertiary");
3705 
3706 			getAst1().populateTuples(tuples);
3707 			tuples.ifFalse(elseexpr);
3708 			getAst2().populateTuples(tuples);
3709 			tuples.gotoAddress(endTertiary);
3710 
3711 			tuples.address(elseexpr);
3712 			getAst3().populateTuples(tuples);
3713 			tuples.address(endTertiary);
3714 
3715 			popSourceLineNumber(tuples);
3716 			return 1;
3717 		}
3718 	}
3719 
3720 	private final class WhileStatementAst extends AST {
3721 
3722 		private Address breakAddress;
3723 		private Address continueAddress;
3724 
3725 		private WhileStatementAst(AST expr, AST block) {
3726 			super(expr, block);
3727 			addFlag(AstFlag.BREAKABLE);
3728 			addFlag(AstFlag.CONTINUEABLE);
3729 		}
3730 
3731 		@Override
3732 		public Address breakAddress() {
3733 			return breakAddress;
3734 		}
3735 
3736 		@Override
3737 		public Address continueAddress() {
3738 			return continueAddress;
3739 		}
3740 
3741 		@Override
3742 		public int populateTuples(AwkTuples tuples) {
3743 			pushSourceLineNumber(tuples);
3744 
3745 			breakAddress = tuples.createAddress("breakAddress");
3746 
3747 			// LOOP
3748 			Address loop = tuples.createAddress("loop");
3749 			tuples.address(loop);
3750 
3751 			// for while statements, the start-of-loop is the continue jump address
3752 			continueAddress = loop;
3753 
3754 			// condition
3755 			getAst1().populateTuples(tuples);
3756 			tuples.ifFalse(breakAddress);
3757 
3758 			if (getAst2() != null) {
3759 				getAst2().populateTuples(tuples);
3760 			}
3761 
3762 			tuples.gotoAddress(loop);
3763 
3764 			tuples.address(breakAddress);
3765 
3766 			popSourceLineNumber(tuples);
3767 			return 0;
3768 		}
3769 	}
3770 
3771 	private final class DoStatementAst extends AST {
3772 
3773 		private Address breakAddress;
3774 		private Address continueAddress;
3775 
3776 		private DoStatementAst(AST block, AST expr) {
3777 			super(block, expr);
3778 			addFlag(AstFlag.BREAKABLE);
3779 			addFlag(AstFlag.CONTINUEABLE);
3780 		}
3781 
3782 		@Override
3783 		public Address breakAddress() {
3784 			return breakAddress;
3785 		}
3786 
3787 		@Override
3788 		public Address continueAddress() {
3789 			return continueAddress;
3790 		}
3791 
3792 		@Override
3793 		public int populateTuples(AwkTuples tuples) {
3794 			pushSourceLineNumber(tuples);
3795 
3796 			breakAddress = tuples.createAddress("breakAddress");
3797 			continueAddress = tuples.createAddress("continueAddress");
3798 
3799 			// LOOP
3800 			Address loop = tuples.createAddress("loop");
3801 			tuples.address(loop);
3802 
3803 			if (getAst1() != null) {
3804 				getAst1().populateTuples(tuples);
3805 			}
3806 
3807 			// for do-while statements, the continue jump address is the loop condition
3808 			tuples.address(continueAddress);
3809 
3810 			// condition
3811 			getAst2().populateTuples(tuples);
3812 			tuples.ifTrue(loop);
3813 
3814 			// tuples.gotoAddress(loop);
3815 
3816 			tuples.address(breakAddress);
3817 
3818 			popSourceLineNumber(tuples);
3819 			return 0;
3820 		}
3821 	}
3822 
3823 	private final class ForStatementAst extends AST {
3824 
3825 		private Address breakAddress;
3826 		private Address continueAddress;
3827 
3828 		private ForStatementAst(AST expr1, AST expr2, AST expr3, AST block) {
3829 			super(expr1, expr2, expr3, block);
3830 			addFlag(AstFlag.BREAKABLE);
3831 			addFlag(AstFlag.CONTINUEABLE);
3832 		}
3833 
3834 		@Override
3835 		public Address breakAddress() {
3836 			return breakAddress;
3837 		}
3838 
3839 		@Override
3840 		public Address continueAddress() {
3841 			return continueAddress;
3842 		}
3843 
3844 		@Override
3845 		public int populateTuples(AwkTuples tuples) {
3846 			pushSourceLineNumber(tuples);
3847 
3848 			breakAddress = tuples.createAddress("breakAddress");
3849 			continueAddress = tuples.createAddress("continueAddress");
3850 
3851 			// initial actions
3852 			if (getAst1() != null) {
3853 				int ast1Result = getAst1().populateTuples(tuples);
3854 				for (int i = 0; i < ast1Result; i++) {
3855 					tuples.pop();
3856 				}
3857 			}
3858 			// LOOP
3859 			Address loop = tuples.createAddress("loop");
3860 			tuples.address(loop);
3861 
3862 			if (getAst2() != null) {
3863 				// condition
3864 				// assert(getAst2() != null);
3865 				getAst2().populateTuples(tuples);
3866 				tuples.ifFalse(breakAddress);
3867 			}
3868 
3869 			if (getAst4() != null) {
3870 				// post loop action
3871 				getAst4().populateTuples(tuples);
3872 			}
3873 
3874 			// for for-loops, the continue jump address is the post-loop-action
3875 			tuples.address(continueAddress);
3876 
3877 			// post-loop action
3878 			if (getAst3() != null) {
3879 				int ast3Result = getAst3().populateTuples(tuples);
3880 				for (int i = 0; i < ast3Result; i++) {
3881 					tuples.pop();
3882 				}
3883 			}
3884 
3885 			tuples.gotoAddress(loop);
3886 
3887 			tuples.address(breakAddress);
3888 
3889 			popSourceLineNumber(tuples);
3890 			return 0;
3891 		}
3892 	}
3893 
3894 	private final class ForInStatementAst extends AST {
3895 
3896 		private Address breakAddress;
3897 		private Address continueAddress;
3898 
3899 		private ForInStatementAst(AST keyIdAst, AST arrayIdAst, AST block) {
3900 			super(keyIdAst, arrayIdAst, block);
3901 			addFlag(AstFlag.BREAKABLE);
3902 			addFlag(AstFlag.CONTINUEABLE);
3903 		}
3904 
3905 		@Override
3906 		public Address breakAddress() {
3907 			return breakAddress;
3908 		}
3909 
3910 		@Override
3911 		public Address continueAddress() {
3912 			return continueAddress;
3913 		}
3914 
3915 		@Override
3916 		public int populateTuples(AwkTuples tuples) {
3917 			pushSourceLineNumber(tuples);
3918 
3919 			breakAddress = tuples.createAddress("breakAddress");
3920 
3921 			populateArrayOperandTuples(getAst2(), tuples, false, getAst2() + " is not an array");
3922 			// pops the array and pushes the keyset
3923 			tuples.keylist();
3924 
3925 			// stack now contains:
3926 			// keylist
3927 
3928 			// LOOP
3929 			Address loop = tuples.createAddress("loop");
3930 			tuples.address(loop);
3931 
3932 			// for for-in loops, the continue jump address is the start-of-loop address
3933 			continueAddress = loop;
3934 
3935 			// condition
3936 			tuples.dup();
3937 			tuples.isEmptyList(breakAddress);
3938 
3939 			// take an element off the set
3940 			tuples.dup();
3941 			tuples.getFirstAndRemoveFromList();
3942 			// assign it to the id
3943 			tuples.assign(((IDAst) getAst1()).offset, ((IDAst) getAst1()).isGlobal);
3944 			tuples.pop(); // remove the assignment result
3945 
3946 			if (getAst3() != null) {
3947 				// execute the block
3948 				getAst3().populateTuples(tuples);
3949 			}
3950 			// otherwise, there is no block to execute
3951 
3952 			tuples.gotoAddress(loop);
3953 
3954 			tuples.address(breakAddress);
3955 			tuples.pop(); // keylist
3956 
3957 			popSourceLineNumber(tuples);
3958 			return 0;
3959 		}
3960 	}
3961 
3962 	@SuppressWarnings("unused")
3963 	private final class EmptyStatementAst extends AST {
3964 
3965 		private EmptyStatementAst() {
3966 			super();
3967 		}
3968 
3969 		@Override
3970 		public int populateTuples(AwkTuples tuples) {
3971 			pushSourceLineNumber(tuples);
3972 			// nothing to populate!
3973 			popSourceLineNumber(tuples);
3974 			return 0;
3975 		}
3976 	}
3977 
3978 	/**
3979 	 * The AST for an expression used as a statement.
3980 	 * If the expression returns a value, the value is popped
3981 	 * off the stack and discarded.
3982 	 */
3983 	private final class ExpressionStatementAst extends AST {
3984 
3985 		private ExpressionStatementAst(AST expr) {
3986 			super(expr);
3987 		}
3988 
3989 		@Override
3990 		public int populateTuples(AwkTuples tuples) {
3991 			pushSourceLineNumber(tuples);
3992 			int exprCount = getAst1().populateTuples(tuples);
3993 			if (exprCount == 1) {
3994 				tuples.popScalar();
3995 			}
3996 			popSourceLineNumber(tuples);
3997 			return 0;
3998 		}
3999 	}
4000 
4001 	private final class AssignmentExpressionAst extends ScalarExpressionAst {
4002 
4003 		/** operand / operator */
4004 		private Token op;
4005 		private String text;
4006 
4007 		private AssignmentExpressionAst(AST lhs, Token op, String text, AST rhs) {
4008 			super(lhs, rhs);
4009 			this.op = op;
4010 			this.text = text;
4011 		}
4012 
4013 		@Override
4014 		public String toString() {
4015 			return super.toString() + " (" + op + "/" + text + ")";
4016 		}
4017 
4018 		@Override
4019 		public int populateTuples(AwkTuples tuples) {
4020 			pushSourceLineNumber(tuples);
4021 			getAst2().populateTuples(tuples); // here, stack contains one value
4022 			if (getAst1() instanceof IDAst) {
4023 				IDAst idAst = (IDAst) getAst1();
4024 				idAst.setScalar(true);
4025 				boolean isSpecial = isJrtManagedSpecialName(idAst.id);
4026 				if (isSpecial) {
4027 					// value is on stack from RHS
4028 					switch (op) {
4029 					case EQUALS:
4030 						assignSpecial(tuples, idAst.id);
4031 						break;
4032 					case PLUS_EQ:
4033 						pushSpecialThenSwap(tuples, idAst.id);
4034 						tuples.add();
4035 						assignSpecial(tuples, idAst.id);
4036 						break;
4037 					case MINUS_EQ:
4038 						pushSpecialThenSwap(tuples, idAst.id);
4039 						tuples.subtract();
4040 						assignSpecial(tuples, idAst.id);
4041 						break;
4042 					case MULT_EQ:
4043 						pushSpecialThenSwap(tuples, idAst.id);
4044 						tuples.multiply();
4045 						assignSpecial(tuples, idAst.id);
4046 						break;
4047 					case DIV_EQ:
4048 						pushSpecialThenSwap(tuples, idAst.id);
4049 						tuples.divide();
4050 						assignSpecial(tuples, idAst.id);
4051 						break;
4052 					case MOD_EQ:
4053 						pushSpecialThenSwap(tuples, idAst.id);
4054 						tuples.mod();
4055 						assignSpecial(tuples, idAst.id);
4056 						break;
4057 					case POW_EQ:
4058 						pushSpecialThenSwap(tuples, idAst.id);
4059 						tuples.pow();
4060 						assignSpecial(tuples, idAst.id);
4061 						break;
4062 					default:
4063 						throw new Error("Unhandled op: " + op + " / " + text);
4064 					}
4065 					if ("RS".equals(idAst.id)) {
4066 						tuples.applyRS();
4067 					}
4068 				} else {
4069 					if (op == Token.EQUALS) {
4070 						// Expected side effect:
4071 						// Upon assignment, if the var is RS, reapply RS to input streams.
4072 						tuples.assign(idAst.offset, idAst.isGlobal);
4073 					} else if (op == Token.PLUS_EQ) {
4074 						tuples.plusEq(idAst.offset, idAst.isGlobal);
4075 					} else if (op == Token.MINUS_EQ) {
4076 						tuples.minusEq(idAst.offset, idAst.isGlobal);
4077 					} else if (op == Token.MULT_EQ) {
4078 						tuples.multEq(idAst.offset, idAst.isGlobal);
4079 					} else if (op == Token.DIV_EQ) {
4080 						tuples.divEq(idAst.offset, idAst.isGlobal);
4081 					} else if (op == Token.MOD_EQ) {
4082 						tuples.modEq(idAst.offset, idAst.isGlobal);
4083 					} else if (op == Token.POW_EQ) {
4084 						tuples.powEq(idAst.offset, idAst.isGlobal);
4085 					} else {
4086 						throw new Error("Unhandled op: " + op + " / " + text);
4087 					}
4088 					if (idAst.id.equals("RS")) {
4089 						tuples.applyRS();
4090 					}
4091 				}
4092 			} else if (getAst1() instanceof ArrayReferenceAst) {
4093 				ArrayReferenceAst arr = (ArrayReferenceAst) getAst1();
4094 				if (arr.getAst1() instanceof IDAst) {
4095 					IDAst idAst = (IDAst) arr.getAst1();
4096 					idAst.setArray(true);
4097 				}
4098 				arr.populateTargetReferenceTuples(tuples);
4099 				if (op == Token.EQUALS) {
4100 					tuples.assignMapElement();
4101 				} else if (op == Token.PLUS_EQ) {
4102 					tuples.plusEqMapElement();
4103 				} else if (op == Token.MINUS_EQ) {
4104 					tuples.minusEqMapElement();
4105 				} else if (op == Token.MULT_EQ) {
4106 					tuples.multEqMapElement();
4107 				} else if (op == Token.DIV_EQ) {
4108 					tuples.divEqMapElement();
4109 				} else if (op == Token.MOD_EQ) {
4110 					tuples.modEqMapElement();
4111 				} else if (op == Token.POW_EQ) {
4112 					tuples.powEqMapElement();
4113 				} else {
4114 					throw new NotImplementedError("Unhandled op: " + op + " / " + text + " for arrays.");
4115 				}
4116 			} else if (getAst1() instanceof DollarExpressionAst) {
4117 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
4118 				dollarExpr.getAst1().populateTuples(tuples); // stack contains eval of dollar arg
4119 
4120 				if (op == Token.EQUALS) {
4121 					tuples.assignAsInputField();
4122 				} else if (op == Token.PLUS_EQ) {
4123 					tuples.plusEqInputField();
4124 				} else if (op == Token.MINUS_EQ) {
4125 					tuples.minusEqInputField();
4126 				} else if (op == Token.MULT_EQ) {
4127 					tuples.multEqInputField();
4128 				} else if (op == Token.DIV_EQ) {
4129 					tuples.divEqInputField();
4130 				} else if (op == Token.MOD_EQ) {
4131 					tuples.modEqInputField();
4132 				} else if (op == Token.POW_EQ) {
4133 					tuples.powEqInputField();
4134 				} else {
4135 					throw new NotImplementedError("Unhandled op: " + op + " / " + text + " for dollar expressions.");
4136 				}
4137 			} else {
4138 				throw new SemanticException("Cannot perform an assignment on: " + getAst1());
4139 			}
4140 			popSourceLineNumber(tuples);
4141 			return 1;
4142 		}
4143 
4144 		private void pushSpecialThenSwap(AwkTuples tuples, String id) {
4145 			pushSpecialVariable(tuples, id);
4146 			tuples.swap();
4147 		}
4148 
4149 		private void assignSpecial(AwkTuples tuples, String id) {
4150 			assignSpecialVariable(tuples, id);
4151 		}
4152 	}
4153 
4154 	private final class InExpressionAst extends ScalarExpressionAst {
4155 
4156 		private InExpressionAst(AST arg, AST arr) {
4157 			super(arg, arr);
4158 		}
4159 
4160 		@Override
4161 		public int populateTuples(AwkTuples tuples) {
4162 			pushSourceLineNumber(tuples);
4163 			if (!(getAst2() instanceof IDAst) && !(getAst2() instanceof ArrayReferenceAst)) {
4164 				throw new SemanticException("Expecting an array for rhs of IN. Got an expression.");
4165 			}
4166 
4167 			// The key is joined and converted with the SUBSEP and CONVFMT in
4168 			// effect when the operator executes, i.e. after the array operand
4169 			// has been evaluated (it may have side effects), just like gawk.
4170 			AST keyAst = getAst1();
4171 			if (keyAst instanceof ArrayIndexAst) {
4172 				ArrayIndexAst indexAst = (ArrayIndexAst) keyAst;
4173 				int count = indexAst.populateComponentTuples(tuples);
4174 				populateArrayOperandTuples(getAst2(), tuples, false, "Expecting an array for rhs of IN. Got a scalar.");
4175 				// Errors while converting the key report where the subscript
4176 				// starts.
4177 				indexAst.pushSourceLineNumber(tuples);
4178 				tuples.applySubsepUnderTop(count);
4179 				indexAst.popSourceLineNumber(tuples);
4180 			} else {
4181 				keyAst.populateTuples(tuples);
4182 				populateArrayOperandTuples(getAst2(), tuples, false, "Expecting an array for rhs of IN. Got a scalar.");
4183 				tuples.applySubsepUnderTop(1);
4184 			}
4185 			tuples.isIn();
4186 
4187 			popSourceLineNumber(tuples);
4188 			return 1;
4189 		}
4190 	}
4191 
4192 	private final class ComparisonExpressionAst extends ScalarExpressionAst {
4193 
4194 		/**
4195 		 * operand / operator
4196 		 */
4197 		private Token op;
4198 		private String text;
4199 
4200 		private ComparisonExpressionAst(AST lhs, Token op, String text, AST rhs) {
4201 			super(lhs, rhs);
4202 			this.op = op;
4203 			this.text = text;
4204 		}
4205 
4206 		@Override
4207 		public String toString() {
4208 			return super.toString() + " (" + op + "/" + text + ")";
4209 		}
4210 
4211 		@Override
4212 		public int populateTuples(AwkTuples tuples) {
4213 			pushSourceLineNumber(tuples);
4214 
4215 			getAst1().populateTuples(tuples);
4216 			if (op == Token.MATCHES || op == Token.NOT_MATCHES) {
4217 				populateRawRegexpParameterTuples(getAst2(), tuples);
4218 			} else {
4219 				getAst2().populateTuples(tuples);
4220 			}
4221 			// 2 values on the stack
4222 
4223 			if (op == Token.EQ) {
4224 				tuples.cmpEq();
4225 			} else if (op == Token.NE) {
4226 				tuples.cmpEq();
4227 				tuples.not();
4228 			} else if (op == Token.LT) {
4229 				tuples.cmpLt();
4230 			} else if (op == Token.GT) {
4231 				tuples.cmpGt();
4232 			} else if (op == Token.LE) {
4233 				tuples.cmpGt();
4234 				tuples.not();
4235 			} else if (op == Token.GE) {
4236 				tuples.cmpLt();
4237 				tuples.not();
4238 			} else if (op == Token.MATCHES) {
4239 				tuples.matches();
4240 			} else if (op == Token.NOT_MATCHES) {
4241 				tuples.matches();
4242 				tuples.not();
4243 			} else {
4244 				throw new Error("Unhandled op: " + op + " / " + text);
4245 			}
4246 
4247 			popSourceLineNumber(tuples);
4248 			return 1;
4249 		}
4250 	}
4251 
4252 	private final class LogicalExpressionAst extends ScalarExpressionAst {
4253 
4254 		/**
4255 		 * operand / operator
4256 		 */
4257 		private Token op;
4258 		private String text;
4259 
4260 		private LogicalExpressionAst(AST lhs, Token op, String text, AST rhs) {
4261 			super(lhs, rhs);
4262 			this.op = op;
4263 			this.text = text;
4264 		}
4265 
4266 		@Override
4267 		public String toString() {
4268 			return super.toString() + " (" + op + "/" + text + ")";
4269 		}
4270 
4271 		@Override
4272 		public int populateTuples(AwkTuples tuples) {
4273 			pushSourceLineNumber(tuples);
4274 			// exhibit short-circuit behavior
4275 			Address end = tuples.createAddress("end");
4276 			getAst1().populateTuples(tuples);
4277 			tuples.dup();
4278 			if (op == Token.OR) {
4279 				// shortCircuit when op is Token.OR and 1st arg is true
4280 				tuples.ifTrue(end);
4281 			} else if (op == Token.AND) {
4282 				tuples.ifFalse(end);
4283 			}
4284 			tuples.pop();
4285 			getAst2().populateTuples(tuples);
4286 			tuples.address(end);
4287 
4288 			// turn the result into boolean one or zero
4289 			tuples.toNumber();
4290 			popSourceLineNumber(tuples);
4291 			return 1;
4292 		}
4293 	}
4294 
4295 	private final class BinaryExpressionAst extends ScalarExpressionAst {
4296 
4297 		/**
4298 		 * operand / operator
4299 		 */
4300 		private Token op;
4301 		private String text;
4302 
4303 		private BinaryExpressionAst(AST lhs, Token op, String text, AST rhs) {
4304 			super(lhs, rhs);
4305 			this.op = op;
4306 			this.text = text;
4307 		}
4308 
4309 		@Override
4310 		public String toString() {
4311 			return super.toString() + " (" + op + "/" + text + ")";
4312 		}
4313 
4314 		@Override
4315 		public int populateTuples(AwkTuples tuples) {
4316 			pushSourceLineNumber(tuples);
4317 			getAst1().populateTuples(tuples);
4318 			getAst2().populateTuples(tuples);
4319 			if (op == Token.PLUS) {
4320 				tuples.add();
4321 			} else if (op == Token.MINUS) {
4322 				tuples.subtract();
4323 			} else if (op == Token.MULT) {
4324 				tuples.multiply();
4325 			} else if (op == Token.DIVIDE) {
4326 				tuples.divide();
4327 			} else if (op == Token.MOD) {
4328 				tuples.mod();
4329 			} else if (op == Token.POW) {
4330 				tuples.pow();
4331 			} else {
4332 				throw new Error("Unhandled op: " + op + " / " + this);
4333 			}
4334 			popSourceLineNumber(tuples);
4335 			return 1;
4336 		}
4337 	}
4338 
4339 	private final class ConcatExpressionAst extends ScalarExpressionAst {
4340 
4341 		private ConcatExpressionAst(AST lhs, AST rhs) {
4342 			super(lhs, rhs);
4343 		}
4344 
4345 		@Override
4346 		public int populateTuples(AwkTuples tuples) {
4347 			pushSourceLineNumber(tuples);
4348 			getAst1().populateTuples(tuples);
4349 			getAst2().populateTuples(tuples);
4350 			tuples.concat();
4351 			popSourceLineNumber(tuples);
4352 			return 1;
4353 		}
4354 	}
4355 
4356 	private final class NegativeExpressionAst extends ScalarExpressionAst {
4357 
4358 		private NegativeExpressionAst(AST expr) {
4359 			super(expr);
4360 		}
4361 
4362 		@Override
4363 		public int populateTuples(AwkTuples tuples) {
4364 			pushSourceLineNumber(tuples);
4365 			getAst1().populateTuples(tuples);
4366 			tuples.negate();
4367 			popSourceLineNumber(tuples);
4368 			return 1;
4369 		}
4370 	}
4371 
4372 	private final class UnaryPlusExpressionAst extends ScalarExpressionAst {
4373 
4374 		private UnaryPlusExpressionAst(AST expr) {
4375 			super(expr);
4376 		}
4377 
4378 		@Override
4379 		public int populateTuples(AwkTuples tuples) {
4380 			pushSourceLineNumber(tuples);
4381 			getAst1().populateTuples(tuples);
4382 			tuples.unaryPlus();
4383 			popSourceLineNumber(tuples);
4384 			return 1;
4385 		}
4386 	}
4387 
4388 	private final class NotExpressionAst extends ScalarExpressionAst {
4389 
4390 		private NotExpressionAst(AST expr) {
4391 			super(expr);
4392 		}
4393 
4394 		@Override
4395 		public int populateTuples(AwkTuples tuples) {
4396 			pushSourceLineNumber(tuples);
4397 			getAst1().populateTuples(tuples);
4398 			tuples.not();
4399 			popSourceLineNumber(tuples);
4400 			return 1;
4401 		}
4402 	}
4403 
4404 	private final class DollarExpressionAst extends ScalarExpressionAst {
4405 
4406 		private DollarExpressionAst(AST expr) {
4407 			super(expr);
4408 		}
4409 
4410 		@Override
4411 		public int populateTuples(AwkTuples tuples) {
4412 			pushSourceLineNumber(tuples);
4413 			getAst1().populateTuples(tuples);
4414 			tuples.getInputField();
4415 			popSourceLineNumber(tuples);
4416 			return 1;
4417 		}
4418 	}
4419 
4420 	private final class ArrayIndexAst extends ScalarExpressionAst {
4421 
4422 		private ArrayIndexAst(int lineNo, AST exprAst, AST next) {
4423 			// The line is captured where the subscript expression starts, so
4424 			// the APPLY_SUBSEP tuple reports errors there.
4425 			super(lineNo, exprAst, next);
4426 		}
4427 
4428 		@Override
4429 		public int populateTuples(AwkTuples tuples) {
4430 			pushSourceLineNumber(tuples);
4431 			int cnt = populateComponentTuples(tuples);
4432 			// Convert the subscript to its string form now, with the CONVFMT in
4433 			// effect at this point of the execution: a later CONVFMT change must
4434 			// not retroactively alter existing keys.
4435 			tuples.applySubsep(cnt);
4436 			popSourceLineNumber(tuples);
4437 			return 1;
4438 		}
4439 
4440 		/**
4441 		 * Evaluates the raw subscript components without joining or converting
4442 		 * them, and returns how many were pushed. The "in" operator uses this
4443 		 * to delay the conversion until its array operand has been evaluated.
4444 		 */
4445 		private int populateComponentTuples(AwkTuples tuples) {
4446 			AST ptr = this;
4447 			int cnt = 0;
4448 			while (ptr != null) {
4449 				ptr.getAst1().populateTuples(tuples);
4450 				++cnt;
4451 				ptr = ptr.getAst2();
4452 			}
4453 			return cnt;
4454 		}
4455 	}
4456 
4457 	// made classname all capitals to stand out in a syntax tree dump
4458 	private final class StatementListAst extends AST {
4459 
4460 		private StatementListAst(AST statementAst, AST rest) {
4461 			super(statementAst, rest);
4462 		}
4463 
4464 		/**
4465 		 * Recursively process statements within this statement list.
4466 		 * <p>
4467 		 * It originally was done linearly. However, quirks in the grammar required
4468 		 * a more general, recursive approach to processing this "list".
4469 		 * <p>
4470 		 * Note: this should be reevaluated periodically in case the grammar
4471 		 * becomes linear again.
4472 		 */
4473 		@Override
4474 		public int populateTuples(AwkTuples tuples) {
4475 			pushSourceLineNumber(tuples);
4476 			// typical recursive processing of a list
4477 			getAst1().populateTuples(tuples);
4478 			if (getAst2() != null) {
4479 				getAst2().populateTuples(tuples);
4480 			}
4481 			popSourceLineNumber(tuples);
4482 			return 0;
4483 		}
4484 
4485 		@Override
4486 		public String toString() {
4487 			return super.toString() + " <" + getAst1() + ">";
4488 		}
4489 	}
4490 
4491 	// made non-static to access the symbol table
4492 	private final class FunctionDefAst extends AST {
4493 
4494 		private String id;
4495 		private Address functionAddress;
4496 		private Address returnAddress;
4497 
4498 		@Override
4499 		public Address returnAddress() {
4500 			return returnAddress;
4501 		}
4502 
4503 		private FunctionDefAst(String id, AST params, AST funcBody) {
4504 			super(params, funcBody);
4505 			this.id = id;
4506 			setFunctionFlag(true);
4507 			addFlag(AstFlag.RETURNABLE);
4508 		}
4509 
4510 		public Address getAddress() {
4511 			return functionAddress;
4512 		}
4513 
4514 		@Override
4515 		public int populateTuples(AwkTuples tuples) {
4516 			pushSourceLineNumber(tuples);
4517 
4518 			functionAddress = tuples.createAddress("function: " + id);
4519 			returnAddress = tuples.createAddress("returnAddress for " + id);
4520 
4521 			// annotate the tuple list
4522 			// (useful for compilation,
4523 			// not necessary for interpretation)
4524 			tuples.function(id, paramCount());
4525 
4526 			// functionAddress refers to first function body statement
4527 			// rather than to function def opcode because during
4528 			// interpretation, the function definition is a nop,
4529 			// and for compilation, the next match of the function
4530 			// name can be used
4531 			tuples.address(functionAddress);
4532 
4533 			// the stack contains the parameters to the function call (in rev order, which is good)
4534 
4535 			// execute the body
4536 			// (function body could be empty [no statements])
4537 			if (getAst2() != null) {
4538 				getAst2().populateTuples(tuples);
4539 			}
4540 
4541 			tuples.address(returnAddress);
4542 
4543 			tuples.returnFromFunction();
4544 
4545 			/////////////////////////////////////////////
4546 
4547 			popSourceLineNumber(tuples);
4548 			return 0;
4549 		}
4550 
4551 		int paramCount() {
4552 			AST ptr = getAst1();
4553 			int count = 0;
4554 			while (ptr != null) {
4555 				++count;
4556 				ptr = ptr.getAst1();
4557 			}
4558 			return count;
4559 		}
4560 
4561 	}
4562 
4563 	private final class FunctionCallAst extends ScalarExpressionAst {
4564 
4565 		private FunctionProxy functionProxy;
4566 
4567 		private FunctionCallAst(FunctionProxy functionProxy, AST params) {
4568 			super(params);
4569 			this.functionProxy = functionProxy;
4570 		}
4571 
4572 		/**
4573 		 * Applies several semantic checks with respect
4574 		 * to user-defined-function calls.
4575 		 * <p>
4576 		 * The checks performed are:
4577 		 * <ul>
4578 		 * <li>Make sure the function is defined.
4579 		 * </ul>
4580 		 * A failure of any one of these checks
4581 		 * results in a SemanticException.
4582 		 *
4583 		 * @throws SemanticException upon a failure of
4584 		 *         any of the semantic checks specified above.
4585 		 */
4586 		@Override
4587 		public void semanticAnalysis() throws SemanticException {
4588 			if (!functionProxy.isDefined()) {
4589 				throw new SemanticException("function " + functionProxy + " not defined");
4590 			}
4591 		}
4592 
4593 		@Override
4594 		public int populateTuples(AwkTuples tuples) {
4595 			pushSourceLineNumber(tuples);
4596 			if (!functionProxy.isDefined()) {
4597 				throw new SemanticException("function " + functionProxy + " not defined");
4598 			}
4599 			tuples.scriptThis();
4600 			int actualParamCountLocal;
4601 			if (getAst1() == null) {
4602 				actualParamCountLocal = 0;
4603 			} else {
4604 				actualParamCountLocal = populateActualParametersUpTo(
4605 						tuples,
4606 						(FunctionCallParamListAst) getAst1(),
4607 						0,
4608 						functionProxy.getFunctionParamCount());
4609 			}
4610 			int formalParamCount = functionProxy.getFunctionParamCount();
4611 
4612 			if (actualParamCount() > formalParamCount) {
4613 				// gawk accepts the call but reports it each time it runs
4614 				tuples.warning(extraArgumentWarning());
4615 			}
4616 			tuples.callFunction(functionProxy, functionProxy.getFunctionName(), formalParamCount, actualParamCountLocal);
4617 			popSourceLineNumber(tuples);
4618 			return 1;
4619 		}
4620 
4621 		private int actualParamCount() {
4622 			int cnt = 0;
4623 			AST ptr = getAst1();
4624 			while (ptr != null) {
4625 				++cnt;
4626 				ptr = ptr.getAst2();
4627 			}
4628 			return cnt;
4629 		}
4630 
4631 		private String extraArgumentWarning() {
4632 			return String.format(
4633 					"gawk: %s:%d: warning: function `%s' called with more arguments than declared",
4634 					sourceBasename(),
4635 					warningLineNo(),
4636 					functionProxy.getFunctionName());
4637 		}
4638 
4639 		private int warningLineNo() {
4640 			if (getAst1() instanceof FunctionCallParamListAst) {
4641 				AST firstParam = getAst1().getAst1();
4642 				if (firstParam != null) {
4643 					return firstParam.getLineNo();
4644 				}
4645 			}
4646 			return getLineNo();
4647 		}
4648 
4649 	}
4650 
4651 	private final class IndirectFunctionCallAst extends ScalarExpressionAst {
4652 
4653 		private IndirectFunctionCallAst(AST functionNameAst, AST params) {
4654 			super(functionNameAst, params);
4655 		}
4656 
4657 		@Override
4658 		public int populateTuples(AwkTuples tuples) {
4659 			pushSourceLineNumber(tuples);
4660 			getAst1().populateTuples(tuples);
4661 			int actualParamCount = populateIndirectActualParameters(tuples, (FunctionCallParamListAst) getAst2());
4662 			tuples.indirectCall(
4663 					symbolTable.indirectFunctionTargets(),
4664 					extensions,
4665 					actualParamCount,
4666 					sourceBasename(),
4667 					getLineNo());
4668 			popSourceLineNumber(tuples);
4669 			return 1;
4670 		}
4671 	}
4672 
4673 	private final class BuiltinFunctionCallAst extends ScalarExpressionAst {
4674 
4675 		private final String id;
4676 		private final BuiltinFunction builtin;
4677 
4678 		private BuiltinFunctionCallAst(String id, AST params) {
4679 			super(params);
4680 			this.id = id;
4681 			this.builtin = BuiltinFunction.of(id);
4682 		}
4683 
4684 		@Override
4685 		public int populateTuples(AwkTuples tuples) {
4686 			pushSourceLineNumber(tuples);
4687 			switch (builtin) {
4688 			case SPRINTF:
4689 				populateSprintfTuples(tuples);
4690 				break;
4691 			case CLOSE:
4692 				populateOneArgumentTuples(tuples, "close");
4693 				tuples.close();
4694 				break;
4695 			case LENGTH:
4696 				populateLengthTuples(tuples);
4697 				break;
4698 			case SRAND:
4699 				populateSrandTuples(tuples);
4700 				break;
4701 			case RAND:
4702 				if (getAst1() != null) {
4703 					throw new SemanticException("rand does not take arguments");
4704 				}
4705 				tuples.rand();
4706 				break;
4707 			case SQRT:
4708 				populateArgumentsTuples(tuples, 1, "sqrt requires only 1 argument");
4709 				tuples.sqrt();
4710 				break;
4711 			case INT:
4712 				populateArgumentsTuples(tuples, 1, "int requires only 1 argument");
4713 				tuples.intFunc();
4714 				break;
4715 			case LOG:
4716 				populateArgumentsTuples(tuples, 1, "log requires only 1 argument");
4717 				tuples.log();
4718 				break;
4719 			case EXP:
4720 				populateArgumentsTuples(tuples, 1, "exp requires only 1 argument");
4721 				tuples.exp();
4722 				break;
4723 			case SIN:
4724 				populateArgumentsTuples(tuples, 1, "sin requires only 1 argument");
4725 				tuples.sin();
4726 				break;
4727 			case COS:
4728 				populateArgumentsTuples(tuples, 1, "cos requires only 1 argument");
4729 				tuples.cos();
4730 				break;
4731 			case ATAN2:
4732 				populateArgumentsTuples(tuples, 2, "atan2 requires 2 arguments");
4733 				tuples.atan2();
4734 				break;
4735 			case MATCH:
4736 				populateMatchArgumentsTuples(tuples);
4737 				tuples.match();
4738 				break;
4739 			case INDEX:
4740 				populateArgumentsTuples(tuples, 2, "index requires 2 arguments");
4741 				tuples.index();
4742 				break;
4743 			case SUB:
4744 			case GSUB:
4745 				populateSubTuples(tuples, builtin == BuiltinFunction.GSUB);
4746 				break;
4747 			case SPLIT:
4748 				populateSplitTuples(tuples);
4749 				break;
4750 			case SUBSTR:
4751 				populateSubstrTuples(tuples);
4752 				break;
4753 			case TOLOWER:
4754 				populateOneArgumentTuples(tuples, "tolower");
4755 				tuples.tolower();
4756 				break;
4757 			case TOUPPER:
4758 				populateOneArgumentTuples(tuples, "toupper");
4759 				tuples.toupper();
4760 				break;
4761 			case SYSTEM:
4762 				populateOneArgumentTuples(tuples, "system");
4763 				tuples.system();
4764 				break;
4765 			default:
4766 				throw new NotImplementedError("builtin: " + id);
4767 			}
4768 			popSourceLineNumber(tuples);
4769 			return 1;
4770 		}
4771 
4772 		/**
4773 		 * Populates the tuples of the argument list, validating that
4774 		 * it produces exactly the expected number of values on the stack.
4775 		 *
4776 		 * @param tuples the tuples to populate
4777 		 * @param expectedCount the exact number of arguments required
4778 		 * @param errorMessage the message of the {@link SemanticException}
4779 		 *        thrown when the argument count does not match
4780 		 */
4781 		private void populateArgumentsTuples(AwkTuples tuples, int expectedCount, String errorMessage) {
4782 			int ast1Result = getAst1().populateTuples(tuples);
4783 			if (ast1Result != expectedCount) {
4784 				throw new SemanticException(errorMessage);
4785 			}
4786 		}
4787 
4788 		/**
4789 		 * Populates the tuples of the argument list for a built-in function
4790 		 * taking exactly one argument, rejecting missing or extra arguments.
4791 		 *
4792 		 * @param tuples the tuples to populate
4793 		 * @param functionName the name of the built-in function, used in
4794 		 *        error messages
4795 		 */
4796 		private void populateOneArgumentTuples(AwkTuples tuples, String functionName) {
4797 			if (getAst1() == null) {
4798 				throw new SemanticException(functionName + " requires 1 argument");
4799 			}
4800 			int ast1Result = getAst1().populateTuples(tuples);
4801 			if (ast1Result != 1) {
4802 				throw new SemanticException(functionName + " requires only 1 argument");
4803 			}
4804 		}
4805 
4806 		/**
4807 		 * Populates the tuples of the argument list for the <code>match</code>
4808 		 * built-in function, treating its 2nd argument as a literal regular
4809 		 * expression.
4810 		 *
4811 		 * @param tuples the tuples to populate
4812 		 */
4813 		private void populateMatchArgumentsTuples(AwkTuples tuples) {
4814 			int ast1Result = populateActualParameters(tuples, (FunctionCallParamListAst) getAst1(), 1);
4815 			if (ast1Result != 2) {
4816 				throw new SemanticException("match requires 2 arguments");
4817 			}
4818 		}
4819 
4820 		/**
4821 		 * Populates the tuples for the <code>sprintf</code> built-in function.
4822 		 *
4823 		 * @param tuples the tuples to populate
4824 		 */
4825 		private void populateSprintfTuples(AwkTuples tuples) {
4826 			if (getAst1() == null) {
4827 				throw new SemanticException("sprintf requires at least 1 argument");
4828 			}
4829 			int ast1Result = getAst1().populateTuples(tuples);
4830 			if (ast1Result == 0) {
4831 				throw new SemanticException("sprintf requires at minimum 1 argument");
4832 			}
4833 			tuples.sprintf(ast1Result);
4834 		}
4835 
4836 		/**
4837 		 * Populates the tuples for the <code>length</code> built-in function,
4838 		 * which takes either no argument (implying $0) or one argument.
4839 		 *
4840 		 * @param tuples the tuples to populate
4841 		 */
4842 		private void populateLengthTuples(AwkTuples tuples) {
4843 			if (getAst1() == null) {
4844 				tuples.length(0);
4845 			} else {
4846 				AST params = getAst1();
4847 				AST argument = params instanceof FunctionCallParamListAst && params.getAst2() == null ?
4848 						params.getAst1() :
4849 						params;
4850 				int ast1Result;
4851 				if (argument instanceof IDAst && !isJrtManagedSpecialName(((IDAst) argument).id)) {
4852 					IDAst idAst = (IDAst) argument;
4853 					tuples.pushIndirectArgument(idAst.offset, idAst.isGlobal);
4854 					ast1Result = 1;
4855 				} else {
4856 					ast1Result = argument.populateTuples(tuples);
4857 				}
4858 				if (ast1Result != 1) {
4859 					throw new SemanticException("length requires at least one argument");
4860 				}
4861 				tuples.length(1);
4862 			}
4863 		}
4864 
4865 		/**
4866 		 * Populates the tuples for the <code>srand</code> built-in function,
4867 		 * which takes either no argument or one argument (the seed).
4868 		 *
4869 		 * @param tuples the tuples to populate
4870 		 */
4871 		private void populateSrandTuples(AwkTuples tuples) {
4872 			if (getAst1() == null) {
4873 				tuples.srand(0);
4874 			} else {
4875 				int ast1Result = getAst1().populateTuples(tuples);
4876 				if (ast1Result != 1) {
4877 					throw new SemanticException("srand takes either 0 or one argument, not " + ast1Result);
4878 				}
4879 				tuples.srand(1);
4880 			}
4881 		}
4882 
4883 		/**
4884 		 * Populates the tuples for the <code>sub</code> and <code>gsub</code>
4885 		 * built-in functions, which take 2 or 3 arguments; the optional 3rd
4886 		 * argument is the substitution target (a variable, an array element,
4887 		 * or an input field reference), defaulting to $0.
4888 		 *
4889 		 * @param tuples the tuples to populate
4890 		 * @param isGsub <code>true</code> for <code>gsub</code> (global
4891 		 *        substitution), <code>false</code> for <code>sub</code>
4892 		 */
4893 		private void populateSubTuples(AwkTuples tuples, boolean isGsub) {
4894 			if (getAst1() == null || getAst1().getAst2() == null || getAst1().getAst2().getAst1() == null) {
4895 				throw new SemanticException("sub needs at least 2 arguments");
4896 			}
4897 			int numargs = 0;
4898 			for (AST paramPtr = getAst1(); paramPtr != null; paramPtr = paramPtr.getAst2()) {
4899 				numargs++;
4900 			}
4901 			if (numargs != 2 && numargs != 3) {
4902 				throw new SemanticException("sub requires 2 or 3 arguments, not " + numargs);
4903 			}
4904 
4905 			populateRawRegexpParameterTuples(getAst1().getAst1(), tuples);
4906 			getAst1().getAst2().getAst1().populateTuples(tuples);
4907 			if (numargs == 3) {
4908 				AST targetAst = getAst1().getAst2().getAst2().getAst1();
4909 				if (targetAst instanceof ArrayReferenceAst) {
4910 					((ArrayReferenceAst) targetAst).populateTargetValueTuples(tuples);
4911 				} else {
4912 					targetAst.populateTuples(tuples);
4913 				}
4914 			}
4915 
4916 			// stack contains arg1,arg2[,arg3] - in that pop() order
4917 
4918 			if (numargs == 2) {
4919 				tuples.subForDollar0(isGsub);
4920 			} else if (numargs == 3) {
4921 				AST ptr = getAst1().getAst2().getAst2().getAst1();
4922 				if (ptr instanceof IDAst) {
4923 					IDAst idAst = (IDAst) ptr;
4924 					idAst.setScalar(true);
4925 					tuples.subForVariable(idAst.offset, idAst.isGlobal, isGsub);
4926 				} else if (ptr instanceof ArrayReferenceAst) {
4927 					ArrayReferenceAst arrAst = (ArrayReferenceAst) ptr;
4928 					if (arrAst.getAst1() instanceof IDAst) {
4929 						IDAst idAst = (IDAst) arrAst.getAst1();
4930 						idAst.setArray(true);
4931 					}
4932 					arrAst.populateTargetReferenceTuples(tuples);
4933 					tuples.subForMapReference(isGsub);
4934 				} else if (ptr instanceof DollarExpressionAst) {
4935 					// push the field ref
4936 					DollarExpressionAst dollarExpr = (DollarExpressionAst) ptr;
4937 					dollarExpr.getAst1().populateTuples(tuples);
4938 					tuples.subForDollarReference(isGsub);
4939 				} else {
4940 					throw new SemanticException(
4941 							"sub's 3rd argument must be either an id, an array reference, or an input field reference");
4942 				}
4943 			}
4944 		}
4945 
4946 		/**
4947 		 * Populates the tuples for the <code>split</code> built-in function.
4948 		 *
4949 		 * @param tuples the tuples to populate
4950 		 */
4951 		private void populateSplitTuples(AwkTuples tuples) {
4952 			// split can take 2 or 3 args:
4953 			// split (string, array [,fs])
4954 			// the 2nd argument is pass by reference, which is ok (?)
4955 
4956 			// funccallparamlist.funccallparamlist.idAst
4957 			if (getAst1() == null || getAst1().getAst2() == null || getAst1().getAst2().getAst1() == null) {
4958 				throw new SemanticException("split needs at least 2 arguments");
4959 			}
4960 			AST ptr = getAst1().getAst2().getAst1();
4961 			if (!(ptr instanceof IDAst) && !(ptr instanceof ArrayReferenceAst)) {
4962 				throw new SemanticException("split needs an array or subarray reference as its 2nd argument");
4963 			}
4964 			if (ptr instanceof IDAst) {
4965 				IDAst arrAst = (IDAst) ptr;
4966 				if (arrAst.isScalar()) {
4967 					throw new SemanticException("split's 2nd arg cannot be a scalar");
4968 				}
4969 				arrAst.setArray(true);
4970 			}
4971 
4972 			int ast1Result = 0;
4973 			for (AST paramPtr = getAst1(); paramPtr != null; paramPtr = paramPtr.getAst2()) {
4974 				ast1Result++;
4975 			}
4976 			if (ast1Result != 2 && ast1Result != 3) {
4977 				throw new SemanticException("split requires 2 or 3 arguments, not " + ast1Result);
4978 			}
4979 
4980 			getAst1().getAst1().populateTuples(tuples);
4981 			populateArrayOperandTuples(ptr, tuples, true, "split's 2nd arg must be an array or subarray reference");
4982 			if (ast1Result == 3) {
4983 				populateRawRegexpParameterTuples(getAst1().getAst2().getAst2().getAst1(), tuples);
4984 			}
4985 			tuples.split(ast1Result);
4986 		}
4987 
4988 		/**
4989 		 * Populates the tuples for the <code>substr</code> built-in function,
4990 		 * which takes 2 or 3 arguments.
4991 		 *
4992 		 * @param tuples the tuples to populate
4993 		 */
4994 		private void populateSubstrTuples(AwkTuples tuples) {
4995 			if (getAst1() == null) {
4996 				throw new SemanticException("substr requires at least 2 arguments");
4997 			}
4998 			int ast1Result = getAst1().populateTuples(tuples);
4999 			if (ast1Result != 2 && ast1Result != 3) {
5000 				throw new SemanticException("substr requires 2 or 3 arguments, not " + ast1Result);
5001 			}
5002 			tuples.substr(ast1Result);
5003 		}
5004 	}
5005 
5006 	private final class FunctionCallParamListAst extends AST {
5007 
5008 		private FunctionCallParamListAst(AST expr, AST rest) {
5009 			super(expr, rest);
5010 		}
5011 
5012 		private FunctionCallParamListAst(int lineNo, AST expr, AST rest) {
5013 			super(lineNo, expr, rest);
5014 		}
5015 
5016 		@Override
5017 		public int populateTuples(AwkTuples tuples) {
5018 			pushSourceLineNumber(tuples);
5019 			int retval;
5020 			if (getAst2() == null) {
5021 				retval = getAst1().populateTuples(tuples);
5022 			} else {
5023 				retval = getAst1().populateTuples(tuples) + getAst2().populateTuples(tuples);
5024 			}
5025 			popSourceLineNumber(tuples);
5026 			return retval;
5027 		}
5028 	}
5029 
5030 	private final class FunctionDefParamListAst extends AST {
5031 
5032 		private String id;
5033 
5034 		private FunctionDefParamListAst(String id, AST rest) {
5035 			super(rest);
5036 			this.id = id;
5037 		}
5038 
5039 		public int populateTuples(AwkTuples tuples) {
5040 			throw new Error("Cannot 'execute' function definition parameter list (formal parameters) in this manner.");
5041 		}
5042 
5043 		/**
5044 		 * According to the spec
5045 		 * (http://www.opengroup.org/onlinepubs/007908799/xcu/awk.html)
5046 		 * formal function parameters cannot be special variables,
5047 		 * such as NF, NR, etc).
5048 		 *
5049 		 * @throws SemanticException upon a semantic error.
5050 		 */
5051 		@Override
5052 		public void semanticAnalysis() throws SemanticException {
5053 			// could do it recursively, but not necessary
5054 			// since all getAst1()'s are FunctionDefParamList's
5055 			// and, thus, terminals (no need to do further
5056 			// semantic analysis)
5057 
5058 			FunctionDefParamListAst ptr = this;
5059 			while (ptr != null) {
5060 				if (isSpecialVariableName(ptr.id)) {
5061 					throw new SemanticException("Special variable " + ptr.id + " cannot be used as a formal parameter");
5062 				}
5063 				ptr = (FunctionDefParamListAst) ptr.getAst1();
5064 			}
5065 		}
5066 	}
5067 
5068 	/**
5069 	 * Flag for non-statement expressions.
5070 	 * Unknown for certain, but I think this is done
5071 	 * to avoid partial variable assignment mistakes.
5072 	 * For example, instead of a=3, the programmer
5073 	 * inadvertently places the a on the line. If IDAsts
5074 	 * were not tagged with AstFlag.NON_STATEMENT, then the
5075 	 * incomplete assignment would parse properly, and
5076 	 * the developer might remain unaware of this issue.
5077 	 */
5078 
5079 	private final class IDAst extends AST {
5080 
5081 		private String id;
5082 		private int offset = AVM.NULL_OFFSET;
5083 		private boolean isGlobal;
5084 		private boolean referenced;
5085 
5086 		private IDAst(String id, boolean isGlobal) {
5087 			this.id = id;
5088 			this.isGlobal = isGlobal;
5089 			addFlag(AstFlag.NON_STATEMENT);
5090 		}
5091 
5092 		private boolean isArray = false;
5093 		private boolean isScalar = false;
5094 
5095 		@Override
5096 		public String toString() {
5097 			return super.toString() + " (" + id + ")";
5098 		}
5099 
5100 		@Override
5101 		public int populateTuples(AwkTuples tuples) {
5102 			pushSourceLineNumber(tuples);
5103 			if (isJrtManagedSpecialName(id)) {
5104 				// Use JRT-managed reads for specials
5105 				pushSpecialVariable(tuples, id);
5106 			} else {
5107 				// Bare identifiers are scalar uses. Array-only contexts emit a
5108 				// typed dereference through populateArrayOperandTuples().
5109 				tuples.dereference(offset, false, isGlobal);
5110 			}
5111 			popSourceLineNumber(tuples);
5112 			return 1;
5113 		}
5114 
5115 		@Override
5116 		public boolean isArray() {
5117 			return isArray;
5118 		}
5119 
5120 		@Override
5121 		public boolean isScalar() {
5122 			return isScalar;
5123 		}
5124 
5125 		private boolean isReferenced() {
5126 			return referenced;
5127 		}
5128 
5129 		private void markReferenced() {
5130 			referenced = true;
5131 		}
5132 
5133 		private void setArray(boolean b) {
5134 			isArray = b;
5135 		}
5136 
5137 		private void setScalar(boolean b) {
5138 			isScalar = b;
5139 		}
5140 	}
5141 
5142 	private final class ArrayReferenceAst extends ScalarExpressionAst {
5143 
5144 		private ArrayReferenceAst(AST idAst, AST idxAst) {
5145 			super(idAst, idxAst);
5146 		}
5147 
5148 		private ArrayReferenceAst(int lineNo, AST idAst, AST idxAst) {
5149 			super(lineNo, idAst, idxAst);
5150 		}
5151 
5152 		@Override
5153 		public String toString() {
5154 			return super.toString() + " (" + getAst1() + " [...])";
5155 		}
5156 
5157 		@Override
5158 		public int populateTuples(AwkTuples tuples) {
5159 			pushSourceLineNumber(tuples);
5160 			// get the containing array, autovivifying missing parent subarrays
5161 			populateContainerTuples(tuples);
5162 			// get the index
5163 			getAst2().populateTuples(tuples);
5164 			tuples.dereferenceArray();
5165 			popSourceLineNumber(tuples);
5166 			return 1;
5167 		}
5168 
5169 		private void populateTargetReferenceTuples(AwkTuples tuples) {
5170 			pushSourceLineNumber(tuples);
5171 			populateContainerTuples(tuples);
5172 			getAst2().populateTuples(tuples);
5173 			popSourceLineNumber(tuples);
5174 		}
5175 
5176 		private void populateArrayValueTuples(AwkTuples tuples, boolean createIfMissing) {
5177 			pushSourceLineNumber(tuples);
5178 			populateContainerTuples(tuples);
5179 			getAst2().populateTuples(tuples);
5180 			if (createIfMissing) {
5181 				tuples.ensureArrayElement();
5182 			} else {
5183 				tuples.peekArrayElement();
5184 			}
5185 			popSourceLineNumber(tuples);
5186 		}
5187 
5188 		private void populateTargetValueTuples(AwkTuples tuples) {
5189 			pushSourceLineNumber(tuples);
5190 			populateContainerTuples(tuples);
5191 			getAst2().populateTuples(tuples);
5192 			tuples.dereferenceArray();
5193 			popSourceLineNumber(tuples);
5194 		}
5195 
5196 		private void populateContainerTuples(AwkTuples tuples) {
5197 			if (getAst1() instanceof ArrayReferenceAst) {
5198 				((ArrayReferenceAst) getAst1()).populateArrayValueTuples(tuples, true);
5199 			} else if (getAst1() instanceof IDAst) {
5200 				IDAst idAst = (IDAst) getAst1();
5201 				if (isJrtManagedSpecialName(idAst.id)) {
5202 					idAst.populateTuples(tuples);
5203 				} else {
5204 					tuples.dereference(idAst.offset, true, idAst.isGlobal);
5205 				}
5206 			} else {
5207 				getAst1().populateTuples(tuples);
5208 			}
5209 		}
5210 	}
5211 
5212 	private final class IntegerAst extends ScalarExpressionAst {
5213 
5214 		private Long value;
5215 
5216 		private IntegerAst(Long value) {
5217 			this.value = value;
5218 			addFlag(AstFlag.NON_STATEMENT);
5219 		}
5220 
5221 		@Override
5222 		public String toString() {
5223 			return super.toString() + " (" + value + ")";
5224 		}
5225 
5226 		@Override
5227 		public int populateTuples(AwkTuples tuples) {
5228 			pushSourceLineNumber(tuples);
5229 			tuples.push(value);
5230 			popSourceLineNumber(tuples);
5231 			return 1;
5232 		}
5233 	}
5234 
5235 	/**
5236 	 * Can either assume the role of a double or an integer
5237 	 * by aggressively normalizing the value to an int if possible.
5238 	 */
5239 	private final class DoubleAst extends ScalarExpressionAst {
5240 
5241 		private Object value;
5242 
5243 		private DoubleAst(Double val) {
5244 			double d = val.doubleValue();
5245 			if (d == (int) d) {
5246 				this.value = (int) d;
5247 			} else {
5248 				this.value = d;
5249 			}
5250 			addFlag(AstFlag.NON_STATEMENT);
5251 		}
5252 
5253 		@Override
5254 		public String toString() {
5255 			return super.toString() + " (" + value + ")";
5256 		}
5257 
5258 		@Override
5259 		public int populateTuples(AwkTuples tuples) {
5260 			pushSourceLineNumber(tuples);
5261 			tuples.push(value);
5262 			popSourceLineNumber(tuples);
5263 			return 1;
5264 		}
5265 	}
5266 
5267 	/**
5268 	 * A string is a string; Awk doesn't attempt to normalize
5269 	 * it until it is used in an arithmetic operation!
5270 	 */
5271 	private final class StringAst extends ScalarExpressionAst {
5272 
5273 		private String value;
5274 
5275 		private StringAst(String str) {
5276 			this.value = str;
5277 			addFlag(AstFlag.NON_STATEMENT);
5278 		}
5279 
5280 		@Override
5281 		public String toString() {
5282 			return super.toString() + " (" + value + ")";
5283 		}
5284 
5285 		@Override
5286 		public int populateTuples(AwkTuples tuples) {
5287 			pushSourceLineNumber(tuples);
5288 			tuples.push(value);
5289 			popSourceLineNumber(tuples);
5290 			return 1;
5291 		}
5292 	}
5293 
5294 	private final class RegexpAst extends ScalarExpressionAst {
5295 
5296 		private String regexpStr;
5297 		private boolean typed;
5298 
5299 		private RegexpAst(String regexpStr, boolean typedParam) {
5300 			this.regexpStr = regexpStr;
5301 			this.typed = typedParam;
5302 		}
5303 
5304 		@Override
5305 		public String toString() {
5306 			return super.toString() + " (" + regexpStr + ")";
5307 		}
5308 
5309 		@Override
5310 		public int populateTuples(AwkTuples tuples) {
5311 			pushSourceLineNumber(tuples);
5312 			if (typed) {
5313 				tuples.regexp(regexpStr);
5314 			} else {
5315 				tuples.getInputField(0);
5316 				tuples.regexp(regexpStr);
5317 				tuples.matches();
5318 			}
5319 			popSourceLineNumber(tuples);
5320 			return 1;
5321 		}
5322 
5323 		private int populateRawRegexpTuples(AwkTuples tuples) {
5324 			tuples.regexp(regexpStr);
5325 			return 1;
5326 		}
5327 	}
5328 
5329 	private final class ConditionPairAst extends ScalarExpressionAst {
5330 
5331 		private final long conditionPairId;
5332 
5333 		private ConditionPairAst(AST booleanAst1, AST booleanAst2) {
5334 			super(booleanAst1, booleanAst2);
5335 			conditionPairId = ++conditionPairCount;
5336 		}
5337 
5338 		@Override
5339 		public int populateTuples(AwkTuples tuples) {
5340 			// The start condition is evaluated only while outside the range, and the
5341 			// end condition only once the range has started (including on the very
5342 			// record that starts it, so a range can begin and end on the same record)
5343 			pushSourceLineNumber(tuples);
5344 
5345 			Address enterRange = tuples.createAddress("conditionPairEnterRange");
5346 			Address testEnd = tuples.createAddress("conditionPairTestEnd");
5347 			Address withinRange = tuples.createAddress("conditionPairWithinRange");
5348 			Address end = tuples.createAddress("conditionPairEnd");
5349 
5350 			tuples.conditionPairInRange(conditionPairId);
5351 			tuples.ifTrue(testEnd);
5352 			getAst1().populateTuples(tuples);
5353 			tuples.ifTrue(enterRange);
5354 			tuples.push(0);
5355 			tuples.gotoAddress(end);
5356 
5357 			tuples.address(enterRange);
5358 			tuples.conditionPairEnter(conditionPairId);
5359 
5360 			tuples.address(testEnd);
5361 			getAst2().populateTuples(tuples);
5362 			tuples.ifFalse(withinRange);
5363 			tuples.conditionPairLeave(conditionPairId);
5364 
5365 			tuples.address(withinRange);
5366 			tuples.push(1);
5367 			tuples.address(end);
5368 
5369 			popSourceLineNumber(tuples);
5370 			return 1;
5371 		}
5372 	}
5373 
5374 	private final class BeginAst extends AST {
5375 
5376 		private BeginAst() {
5377 			super();
5378 			setBeginFlag(true);
5379 		}
5380 
5381 		@Override
5382 		public int populateTuples(AwkTuples tuples) {
5383 			pushSourceLineNumber(tuples);
5384 			tuples.push(1);
5385 			popSourceLineNumber(tuples);
5386 			return 1;
5387 		}
5388 	}
5389 
5390 	private final class EndAst extends AST {
5391 
5392 		private EndAst() {
5393 			super();
5394 			setEndFlag(true);
5395 		}
5396 
5397 		@Override
5398 		public int populateTuples(AwkTuples tuples) {
5399 			pushSourceLineNumber(tuples);
5400 			tuples.push(1);
5401 			popSourceLineNumber(tuples);
5402 			return 1;
5403 		}
5404 	}
5405 
5406 	private final class BeginFileAst extends AST {
5407 
5408 		private BeginFileAst() {
5409 			super();
5410 			setBeginFileFlag(true);
5411 		}
5412 
5413 		@Override
5414 		public int populateTuples(AwkTuples tuples) {
5415 			pushSourceLineNumber(tuples);
5416 			tuples.push(1);
5417 			popSourceLineNumber(tuples);
5418 			return 1;
5419 		}
5420 	}
5421 
5422 	private final class EndFileAst extends AST {
5423 
5424 		private EndFileAst() {
5425 			super();
5426 			setEndFileFlag(true);
5427 		}
5428 
5429 		@Override
5430 		public int populateTuples(AwkTuples tuples) {
5431 			pushSourceLineNumber(tuples);
5432 			tuples.push(1);
5433 			popSourceLineNumber(tuples);
5434 			return 1;
5435 		}
5436 	}
5437 
5438 	private final class PreIncAst extends ScalarExpressionAst {
5439 
5440 		private PreIncAst(AST symbolAst) {
5441 			super(symbolAst);
5442 		}
5443 
5444 		@Override
5445 		public int populateTuples(AwkTuples tuples) {
5446 			pushSourceLineNumber(tuples);
5447 			if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
5448 				// the sequence already leaves the new value on the stack
5449 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, true, false);
5450 				popSourceLineNumber(tuples);
5451 				return 1;
5452 			} else if (getAst1() instanceof IDAst) {
5453 				IDAst idAst = (IDAst) getAst1();
5454 				tuples.inc(idAst.offset, idAst.isGlobal);
5455 			} else if (getAst1() instanceof ArrayReferenceAst) {
5456 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
5457 				if (arrAst.getAst1() instanceof IDAst) {
5458 					IDAst idAst = (IDAst) arrAst.getAst1();
5459 					if (idAst.isScalar()) {
5460 						throw new SemanticException("Cannot use " + idAst + " as an array.");
5461 					}
5462 					idAst.setArray(true);
5463 				}
5464 				arrAst.populateTargetReferenceTuples(tuples);
5465 				tuples.incMapRef();
5466 			} else if (getAst1() instanceof DollarExpressionAst) {
5467 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
5468 				dollarExpr.getAst1().populateTuples(tuples); // OPTIMIATION: duplicate the x in $x here
5469 				// so that it is not evaluated again
5470 				tuples.dup();
5471 				// stack contains eval of dollar arg
5472 				// tuples.assignAsInputField();
5473 				tuples.incDollarRef();
5474 				// OPTIMIATION continued: now evaluate
5475 				// the dollar expression with x (for $x)
5476 				// instead of evaluating the expression again
5477 				tuples.getInputField();
5478 				popSourceLineNumber(tuples);
5479 				return 1; // NOTE, short-circuit return here!
5480 			} else {
5481 				throw new NotImplementedError("unhandled preinc for " + getAst1());
5482 			}
5483 			// else
5484 			// assert false : "cannot refer for preInc to "+getAst1();
5485 			getAst1().populateTuples(tuples);
5486 			popSourceLineNumber(tuples);
5487 			return 1;
5488 		}
5489 	}
5490 
5491 	private final class PreDecAst extends ScalarExpressionAst {
5492 
5493 		private PreDecAst(AST symbolAst) {
5494 			super(symbolAst);
5495 		}
5496 
5497 		@Override
5498 		public int populateTuples(AwkTuples tuples) {
5499 			pushSourceLineNumber(tuples);
5500 			if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
5501 				// the sequence already leaves the new value on the stack
5502 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, false, false);
5503 				popSourceLineNumber(tuples);
5504 				return 1;
5505 			} else if (getAst1() instanceof IDAst) {
5506 				IDAst idAst = (IDAst) getAst1();
5507 				tuples.dec(idAst.offset, idAst.isGlobal);
5508 			} else if (getAst1() instanceof ArrayReferenceAst) {
5509 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
5510 				if (arrAst.getAst1() instanceof IDAst) {
5511 					IDAst idAst = (IDAst) arrAst.getAst1();
5512 					if (idAst.isScalar()) {
5513 						throw new SemanticException("Cannot use " + idAst + " as an array.");
5514 					}
5515 					idAst.setArray(true);
5516 				}
5517 				arrAst.populateTargetReferenceTuples(tuples);
5518 				tuples.decMapRef();
5519 			} else if (getAst1() instanceof DollarExpressionAst) {
5520 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
5521 				dollarExpr.getAst1().populateTuples(tuples); // OPTIMIATION: duplicate the x in $x here
5522 				// so that it is not evaluated again
5523 				tuples.dup();
5524 				// stack contains eval of dollar arg
5525 				// tuples.assignAsInputField();
5526 				tuples.decDollarRef();
5527 				// OPTIMIATION continued: now evaluate
5528 				// the dollar expression with x (for $x)
5529 				// instead of evaluating the expression again
5530 				tuples.getInputField();
5531 				popSourceLineNumber(tuples);
5532 				return 1; // NOTE, short-circuit return here!
5533 			} else {
5534 				throw new NotImplementedError("unhandled predec for " + getAst1());
5535 			}
5536 			getAst1().populateTuples(tuples);
5537 			popSourceLineNumber(tuples);
5538 			return 1;
5539 		}
5540 	}
5541 
5542 	private final class PostIncAst extends ScalarExpressionAst {
5543 
5544 		private PostIncAst(AST symbolAst) {
5545 			super(symbolAst);
5546 		}
5547 
5548 		@Override
5549 		public int populateTuples(AwkTuples tuples) {
5550 			pushSourceLineNumber(tuples);
5551 			if (getAst1() instanceof DollarExpressionAst) {
5552 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
5553 				dollarExpr.getAst1().populateTuples(tuples);
5554 				tuples.incDollarRef();
5555 			} else if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
5556 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, true, true);
5557 			} else {
5558 				if (getAst1() instanceof ArrayReferenceAst) {
5559 					((ArrayReferenceAst) getAst1()).populateTargetValueTuples(tuples);
5560 					tuples.unaryPlus();
5561 				} else {
5562 					getAst1().populateTuples(tuples);
5563 				}
5564 				if (getAst1() instanceof IDAst) {
5565 					IDAst idAst = (IDAst) getAst1();
5566 					tuples.postInc(idAst.offset, idAst.isGlobal);
5567 				} else if (getAst1() instanceof ArrayReferenceAst) {
5568 					ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
5569 					if (arrAst.getAst1() instanceof IDAst) {
5570 						IDAst idAst = (IDAst) arrAst.getAst1();
5571 						if (idAst.isScalar()) {
5572 							throw new SemanticException("Cannot use " + idAst + " as an array.");
5573 						}
5574 						idAst.setArray(true);
5575 					}
5576 					arrAst.populateTargetReferenceTuples(tuples);
5577 					tuples.incMapRef();
5578 				} else {
5579 					throw new NotImplementedError("unhandled postinc for " + getAst1());
5580 				}
5581 			}
5582 			popSourceLineNumber(tuples);
5583 			return 1;
5584 		}
5585 	}
5586 
5587 	private final class PostDecAst extends ScalarExpressionAst {
5588 
5589 		private PostDecAst(AST symbolAst) {
5590 			super(symbolAst);
5591 		}
5592 
5593 		@Override
5594 		public int populateTuples(AwkTuples tuples) {
5595 			pushSourceLineNumber(tuples);
5596 			if (getAst1() instanceof IDAst && isJrtManagedSpecialName(((IDAst) getAst1()).id)) {
5597 				populateSpecialIncDec(tuples, ((IDAst) getAst1()).id, false, true);
5598 				popSourceLineNumber(tuples);
5599 				return 1;
5600 			}
5601 			if (getAst1() instanceof ArrayReferenceAst) {
5602 				((ArrayReferenceAst) getAst1()).populateTargetValueTuples(tuples);
5603 				tuples.unaryPlus();
5604 			} else {
5605 				getAst1().populateTuples(tuples);
5606 			}
5607 			if (getAst1() instanceof IDAst) {
5608 				IDAst idAst = (IDAst) getAst1();
5609 				tuples.postDec(idAst.offset, idAst.isGlobal);
5610 			} else if (getAst1() instanceof ArrayReferenceAst) {
5611 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
5612 				if (arrAst.getAst1() instanceof IDAst) {
5613 					IDAst idAst = (IDAst) arrAst.getAst1();
5614 					if (idAst.isScalar()) {
5615 						throw new SemanticException("Cannot use " + idAst + " as an array.");
5616 					}
5617 					idAst.setArray(true);
5618 				}
5619 				arrAst.populateTargetReferenceTuples(tuples);
5620 				tuples.decMapRef();
5621 			} else if (getAst1() instanceof DollarExpressionAst) {
5622 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst1();
5623 				dollarExpr.getAst1().populateTuples(tuples);
5624 				tuples.decDollarRef();
5625 			} else {
5626 				throw new NotImplementedError("unhandled postinc for " + getAst1());
5627 			}
5628 			popSourceLineNumber(tuples);
5629 			return 1;
5630 		}
5631 	}
5632 
5633 	private final class PrintAst extends ScalarExpressionAst {
5634 
5635 		private Token outputToken;
5636 		private boolean parenthesized;
5637 
5638 		private PrintAst(AST exprList, Token outToken, AST outputExpr, boolean parenthesized) {
5639 			super(exprList, outputExpr);
5640 			this.outputToken = outToken;
5641 			this.parenthesized = parenthesized;
5642 		}
5643 
5644 		@Override
5645 		public int populateTuples(AwkTuples tuples) {
5646 			pushSourceLineNumber(tuples);
5647 
5648 			int paramCount;
5649 			if (getAst1() == null) {
5650 				if (parenthesized) {
5651 					throw new SemanticException("print() requires at least 1 argument");
5652 				}
5653 				paramCount = 0;
5654 			} else {
5655 				paramCount = getAst1().populateTuples(tuples);
5656 				if (paramCount == 0) {
5657 					throw new SemanticException("Cannot print the result. The expression doesn't return anything.");
5658 				}
5659 			}
5660 
5661 			if (getAst2() != null) {
5662 				getAst2().populateTuples(tuples);
5663 			}
5664 
5665 			if (outputToken == Token.GT) {
5666 				tuples.printToFile(paramCount, false); // false = no append
5667 			} else if (outputToken == Token.APPEND) {
5668 				tuples.printToFile(paramCount, true); // false = no append
5669 			} else if (outputToken == Token.PIPE) {
5670 				tuples.printToPipe(paramCount);
5671 			} else {
5672 				tuples.print(paramCount);
5673 			}
5674 
5675 			popSourceLineNumber(tuples);
5676 			return 0;
5677 		}
5678 	}
5679 
5680 	// we don't know if it is a scalar
5681 	/**
5682 	 * Returns whether the identifier names a JRT-managed special variable,
5683 	 * read and written through dedicated opcodes instead of a global slot.
5684 	 * ENVIRON and ARGV are excluded: they are special names but plain
5685 	 * slot-backed arrays, materialized by the preamble. SYMTAB and FUNCTAB
5686 	 * need no exclusion because they are not special names at all: they are
5687 	 * ordinary globals that the runtime populates when the script references
5688 	 * them.
5689 	 */
5690 	private boolean isJrtManagedSpecialName(String id) {
5691 		return isSpecialVariableName(id) && !"ENVIRON".equals(id) && !"ARGV".equals(id);
5692 	}
5693 
5694 	/**
5695 	 * Returns whether the identifier names a special variable in the current
5696 	 * compile-time mode. Most special names (NR, FS, FILENAME, ...) always
5697 	 * are. The gawk-only ERRNO and ARGIND are special outside POSIX mode
5698 	 * only: with {@code --posix} they are plain identifiers — usable as
5699 	 * function parameters and compiled as ordinary globals — exactly like
5700 	 * {@code gawk --posix} treats them.
5701 	 *
5702 	 * @param id the identifier to inspect
5703 	 * @return {@code true} when the name is special in the current mode
5704 	 */
5705 	private boolean isSpecialVariableName(String id) {
5706 		if (!SPECIAL_VAR_NAMES.containsKey(id)) {
5707 			return false;
5708 		}
5709 		if (JRT.isGawkOnlySpecialVariable(id)) {
5710 			return !posix;
5711 		}
5712 		return true;
5713 	}
5714 
5715 	/** Emits the tuple pushing the value of a JRT-managed special variable. */
5716 	private void pushSpecialVariable(AwkTuples tuples, String id) {
5717 		switch (id) {
5718 		case "NF":
5719 			tuples.pushNF();
5720 			break;
5721 		case "NR":
5722 			tuples.pushNR();
5723 			break;
5724 		case "FNR":
5725 			tuples.pushFNR();
5726 			break;
5727 		case "FS":
5728 			tuples.pushFS();
5729 			break;
5730 		case "RS":
5731 			tuples.pushRS();
5732 			break;
5733 		case "OFS":
5734 			tuples.pushOFS();
5735 			break;
5736 		case "ORS":
5737 			tuples.pushORS();
5738 			break;
5739 		case "RSTART":
5740 			tuples.pushRSTART();
5741 			break;
5742 		case "RLENGTH":
5743 			tuples.pushRLENGTH();
5744 			break;
5745 		case "IGNORECASE":
5746 			tuples.pushIGNORECASE();
5747 			break;
5748 		case "FILENAME":
5749 			tuples.pushFILENAME();
5750 			break;
5751 		case "SUBSEP":
5752 			tuples.pushSUBSEP();
5753 			break;
5754 		case "CONVFMT":
5755 			tuples.pushCONVFMT();
5756 			break;
5757 		case "OFMT":
5758 			tuples.pushOFMT();
5759 			break;
5760 		case "ARGC":
5761 			tuples.pushARGC();
5762 			break;
5763 		case "ERRNO":
5764 			tuples.pushERRNO();
5765 			break;
5766 		case "ARGIND":
5767 			tuples.pushARGIND();
5768 			break;
5769 		default:
5770 			throw new Error("Unhandled special var: " + id);
5771 		}
5772 	}
5773 
5774 	/** Emits the tuple assigning the top of the stack to a JRT-managed special variable. */
5775 	private void assignSpecialVariable(AwkTuples tuples, String id) {
5776 		switch (id) {
5777 		case "NF":
5778 			tuples.assignNF();
5779 			break;
5780 		case "NR":
5781 			tuples.assignNR();
5782 			break;
5783 		case "FNR":
5784 			tuples.assignFNR();
5785 			break;
5786 		case "FS":
5787 			tuples.assignFS();
5788 			break;
5789 		case "RS":
5790 			tuples.assignRS();
5791 			break;
5792 		case "OFS":
5793 			tuples.assignOFS();
5794 			break;
5795 		case "ORS":
5796 			tuples.assignORS();
5797 			break;
5798 		case "RSTART":
5799 			tuples.assignRSTART();
5800 			break;
5801 		case "RLENGTH":
5802 			tuples.assignRLENGTH();
5803 			break;
5804 		case "IGNORECASE":
5805 			tuples.assignIGNORECASE();
5806 			break;
5807 		case "FILENAME":
5808 			tuples.assignFILENAME();
5809 			break;
5810 		case "SUBSEP":
5811 			tuples.assignSUBSEP();
5812 			break;
5813 		case "CONVFMT":
5814 			tuples.assignCONVFMT();
5815 			break;
5816 		case "OFMT":
5817 			tuples.assignOFMT();
5818 			break;
5819 		case "ARGC":
5820 			tuples.assignARGC();
5821 			break;
5822 		case "ERRNO":
5823 			tuples.assignERRNO();
5824 			break;
5825 		case "ARGIND":
5826 			tuples.assignARGIND();
5827 			break;
5828 		default:
5829 			throw new Error("Unhandled special var: " + id);
5830 		}
5831 	}
5832 
5833 	/*
5834 	 * Increments or decrements a JRT-managed special variable. Specials live in
5835 	 * the JRT rather than in a global slot, so the slot-based INC/DEC opcodes
5836 	 * cannot be used; the sequence below reads, adjusts, and assigns through
5837 	 * the special-variable opcodes, leaving the expression value (old value for
5838 	 * postfix, new value for prefix) on the stack.
5839 	 */
5840 	private void populateSpecialIncDec(AwkTuples tuples, String id, boolean increment, boolean postfix) {
5841 		pushSpecialVariable(tuples, id);
5842 		if (postfix) {
5843 			tuples.dup();
5844 		}
5845 		tuples.push(Long.valueOf(1L));
5846 		if (increment) {
5847 			tuples.add();
5848 		} else {
5849 			tuples.subtract();
5850 		}
5851 		assignSpecialVariable(tuples, id);
5852 		if (postfix) {
5853 			tuples.pop();
5854 		}
5855 	}
5856 
5857 	/*
5858 	 * A call-expression node is normally stamped when it is reduced, after the
5859 	 * lexer may already have consumed the line terminator; the first argument
5860 	 * carries the line of the call site itself, which is what gawk-style
5861 	 * diagnostics report.
5862 	 */
5863 	private int extensionCallLineNumber(AST params) {
5864 		if (params != null && params.getAst1() != null) {
5865 			return params.getAst1().getLineNo();
5866 		}
5867 		return currentSourceLineNumber();
5868 	}
5869 
5870 	private final class ExtensionAst extends AST {
5871 
5872 		private final ExtensionFunction function;
5873 
5874 		private ExtensionAst(ExtensionFunction functionParam, AST paramAst, int lineNoParam) {
5875 			super(lineNoParam, paramAst);
5876 			this.function = functionParam;
5877 		}
5878 
5879 		@Override
5880 		public int populateTuples(AwkTuples tuples) {
5881 			pushSourceLineNumber(tuples);
5882 			int argCount;
5883 			if (getAst1() == null) {
5884 				argCount = 0;
5885 			} else {
5886 				argCount = countParams((FunctionCallParamListAst) getAst1());
5887 			}
5888 
5889 			int[] reqArrayIdxs = function.collectAssocArrayIndexes(argCount);
5890 			int[] rawValueIdxs = function.collectRawValueIndexes(argCount);
5891 
5892 			int paramCount;
5893 			if (getAst1() == null) {
5894 				paramCount = 0;
5895 			} else {
5896 				Set<Integer> arrayIndexes = new HashSet<Integer>();
5897 				Set<Integer> rawValueIndexes = new HashSet<Integer>();
5898 				Set<Integer> literalRegexpIndexes = new HashSet<Integer>();
5899 				for (int idx : rawValueIdxs) {
5900 					rawValueIndexes.add(Integer.valueOf(idx));
5901 				}
5902 				for (int idx : reqArrayIdxs) {
5903 					AST paramAst = getParamAst((FunctionCallParamListAst) getAst1(), idx).getAst1();
5904 					if (paramAst instanceof IDAst) {
5905 						IDAst idAst = (IDAst) paramAst;
5906 						if (idAst.isScalar()) {
5907 							throw new SemanticException(
5908 									"Extension '" + function.getKeyword() + "' requires parameter position " + idx
5909 											+ " be an associative array, not a scalar.");
5910 						}
5911 						idAst.setArray(true);
5912 						arrayIndexes.add(Integer.valueOf(idx));
5913 					} else if (paramAst instanceof ArrayReferenceAst) {
5914 						arrayIndexes.add(Integer.valueOf(idx));
5915 					}
5916 				}
5917 				for (int idx : function.collectRegexpIndexes(argCount)) {
5918 					literalRegexpIndexes.add(Integer.valueOf(idx));
5919 				}
5920 
5921 				paramCount = populateActualParameters(
5922 						tuples,
5923 						(FunctionCallParamListAst) getAst1(),
5924 						arrayIndexes,
5925 						rawValueIndexes,
5926 						literalRegexpIndexes,
5927 						0);
5928 			}
5929 			tuples.extension(function, paramCount);
5930 			popSourceLineNumber(tuples);
5931 			// an extension always returns a value, even if it is blank/null
5932 			return 1;
5933 		}
5934 
5935 		private AST getParamAst(FunctionCallParamListAst pAst, int pos) {
5936 			for (int i = 0; i < pos; ++i) {
5937 				pAst = (FunctionCallParamListAst) pAst.getAst2();
5938 				if (pAst == null) {
5939 					throw new SemanticException("More arguments required for assoc array parameter position specification.");
5940 				}
5941 			}
5942 			return pAst;
5943 		}
5944 
5945 		private int countParams(FunctionCallParamListAst pAst) {
5946 			int cnt = 0;
5947 			while (pAst != null) {
5948 				pAst = (FunctionCallParamListAst) pAst.getAst2();
5949 				++cnt;
5950 			}
5951 			return cnt;
5952 		}
5953 
5954 		@Override
5955 		public String toString() {
5956 			return super.toString() + " (" + function.getKeyword() + ")";
5957 		}
5958 	}
5959 
5960 	private final class PrintfAst extends ScalarExpressionAst {
5961 
5962 		private Token outputToken;
5963 
5964 		private PrintfAst(AST exprList, Token outToken, AST outputExpr) {
5965 			super(exprList, outputExpr);
5966 			this.outputToken = outToken;
5967 		}
5968 
5969 		@Override
5970 		public int populateTuples(AwkTuples tuples) {
5971 			pushSourceLineNumber(tuples);
5972 
5973 			int paramCount;
5974 			if (getAst1() == null) {
5975 				throw new SemanticException("printf requires at least 1 argument");
5976 			} else {
5977 				paramCount = getAst1().populateTuples(tuples);
5978 				if (paramCount == 0) {
5979 					throw new SemanticException("Cannot printf the result. The expression doesn't return anything.");
5980 				}
5981 			}
5982 
5983 			if (getAst2() != null) {
5984 				getAst2().populateTuples(tuples);
5985 			}
5986 
5987 			if (outputToken == Token.GT) {
5988 				tuples.printfToFile(paramCount, false); // false = no append
5989 			} else if (outputToken == Token.APPEND) {
5990 				tuples.printfToFile(paramCount, true); // false = no append
5991 			} else if (outputToken == Token.PIPE) {
5992 				tuples.printfToPipe(paramCount);
5993 			} else {
5994 				tuples.printf(paramCount);
5995 			}
5996 
5997 			popSourceLineNumber(tuples);
5998 			return 0;
5999 		}
6000 	}
6001 
6002 	private final class GetlineAst extends ScalarExpressionAst {
6003 
6004 		private GetlineAst(AST pipeExpr, AST lvalueAst, AST inRedirect) {
6005 			super(pipeExpr, lvalueAst, inRedirect);
6006 		}
6007 
6008 		@Override
6009 		public int populateTuples(AwkTuples tuples) {
6010 			pushSourceLineNumber(tuples);
6011 			// gawk restriction: only redirected forms of getline may be used
6012 			// inside BEGINFILE/ENDFILE rules. Direct uses are rejected here at
6013 			// compile time; uses reached through user-defined functions are
6014 			// caught at runtime by the interpreter, which knows the current
6015 			// rule.
6016 			if (getAst1() == null && getAst3() == null) {
6017 				AST enclosingRule = searchFor(AstFlag.NEXTABLE);
6018 				AST pattern = enclosingRule == null ? null : enclosingRule.getAst1();
6019 				if (pattern != null && (pattern.isBeginFile() || pattern.isEndFile())) {
6020 					throw new SemanticException(
6021 							"non-redirected `getline' invalid inside `" + (pattern.isBeginFile() ? "BEGINFILE" : "ENDFILE")
6022 									+ "' rule");
6023 				}
6024 			}
6025 			if (getAst1() == null && getAst3() == null && getAst2() == null) {
6026 				tuples.getlineInput();
6027 				popSourceLineNumber(tuples);
6028 				return 1;
6029 			}
6030 			// When no record is read (EOF or, for the redirected forms, an I/O
6031 			// error), the read opcode pushes only the return code and jumps
6032 			// here, past the assignment, so the target keeps its previous
6033 			// value, as POSIX and gawk require.
6034 			Address noRecord = tuples.createAddress("getline_no_record");
6035 			if (getAst1() != null) {
6036 				getAst1().populateTuples(tuples);// stack has getAst1() (i.e., "command")
6037 				tuples.useAsCommandInput(noRecord);
6038 			} else if (getAst3() != null) {
6039 // getline ... < getAst3()
6040 				getAst3().populateTuples(tuples); // stack has getAst3() (i.e., "filename")
6041 				tuples.useAsFileInput(noRecord);
6042 			} else {
6043 				tuples.getlineInputToTarget(noRecord);
6044 			}
6045 			// A record was read: 2 resultant values on the stack!
6046 			// 2nd - 1 for success
6047 			// 1st(top) - the input
6048 			if (getAst2() == null) {
6049 				tuples.assignAsInput();
6050 				// stack still has the input, to be popped below...
6051 				// (all assignment results are placed on the stack)
6052 			} else if (getAst2() instanceof IDAst) {
6053 				IDAst idAst = (IDAst) getAst2();
6054 				tuples.assign(idAst.offset, idAst.isGlobal);
6055 				if (idAst.id.equals("RS")) {
6056 					tuples.applyRS();
6057 				}
6058 			} else if (getAst2() instanceof ArrayReferenceAst) {
6059 				ArrayReferenceAst arr = (ArrayReferenceAst) getAst2();
6060 				if (arr.getAst1() instanceof IDAst) {
6061 					IDAst idAst = (IDAst) arr.getAst1();
6062 					if (idAst.isScalar()) {
6063 						throw new SemanticException("Cannot use " + idAst + " as an array.");
6064 					}
6065 					idAst.setArray(true);
6066 				}
6067 				arr.populateTargetReferenceTuples(tuples);
6068 				tuples.assignMapElement();
6069 			} else if (getAst2() instanceof DollarExpressionAst) {
6070 				DollarExpressionAst dollarExpr = (DollarExpressionAst) getAst2();
6071 				// stack gets the eval of the dollar arg, kept in ast1
6072 				dollarExpr.getAst1().populateTuples(tuples);
6073 				tuples.assignAsInputField();
6074 			} else {
6075 				throw new SemanticException("Cannot getline into a " + getAst2());
6076 			}
6077 			// get rid of value left by the assignment
6078 			tuples.pop();
6079 			if (getAst2() instanceof ArrayReferenceAst || getAst2() instanceof DollarExpressionAst) {
6080 				// gawk evaluates the subscript of an array or field target even
6081 				// when nothing was read — and the mere reference creates the
6082 				// array element — a dark corner the getline5 gawk test pins.
6083 				// Replay that evaluation on the no-record path.
6084 				Address end = tuples.createAddress("getline_end");
6085 				tuples.gotoAddress(end);
6086 				tuples.address(noRecord);
6087 				// Evaluating the target as a value creates an array element and
6088 				// rejects an invalid field number, exactly like the assignment
6089 				// would, without storing anything.
6090 				getAst2().populateTuples(tuples);
6091 				tuples.pop();
6092 				tuples.address(end);
6093 			} else {
6094 				tuples.address(noRecord);
6095 			}
6096 			// one value (the return code) is left on the stack
6097 			popSourceLineNumber(tuples);
6098 			return 1;
6099 		}
6100 	}
6101 
6102 	private final class ReturnStatementAst extends AST {
6103 
6104 		private ReturnStatementAst(AST expr) {
6105 			super(expr);
6106 		}
6107 
6108 		@Override
6109 		public int populateTuples(AwkTuples tuples) {
6110 			pushSourceLineNumber(tuples);
6111 			AST returnable = searchFor(AstFlag.RETURNABLE);
6112 			if (returnable == null) {
6113 				throw new SemanticException("Cannot use return here.");
6114 			}
6115 			if (getAst1() != null) {
6116 				getAst1().populateTuples(tuples);
6117 				tuples.setReturnResult();
6118 			}
6119 			tuples.gotoAddress(returnable.returnAddress());
6120 			popSourceLineNumber(tuples);
6121 			return 0;
6122 		}
6123 	}
6124 
6125 	private final class ExitStatementAst extends AST {
6126 
6127 		private ExitStatementAst(AST expr) {
6128 			super(expr);
6129 		}
6130 
6131 		@Override
6132 		public int populateTuples(AwkTuples tuples) {
6133 			pushSourceLineNumber(tuples);
6134 			if (getAst1() != null) {
6135 				getAst1().populateTuples(tuples);
6136 				tuples.exitWithCode();
6137 			} else {
6138 				tuples.exitWithoutCode();
6139 			}
6140 			popSourceLineNumber(tuples);
6141 			return 0;
6142 		}
6143 	}
6144 
6145 	private final class DeleteStatementAst extends AST {
6146 
6147 		private DeleteStatementAst(AST symbolAst) {
6148 			super(symbolAst);
6149 		}
6150 
6151 		@Override
6152 		public int populateTuples(AwkTuples tuples) {
6153 			pushSourceLineNumber(tuples);
6154 
6155 			if (getAst1() instanceof ArrayReferenceAst) {
6156 				ArrayReferenceAst arrAst = (ArrayReferenceAst) getAst1();
6157 				if (arrAst.getAst1() instanceof IDAst) {
6158 					IDAst idAst = (IDAst) arrAst.getAst1();
6159 					idAst.setArray(true);
6160 				}
6161 				arrAst.populateTargetReferenceTuples(tuples);
6162 				tuples.deleteMapElement();
6163 			} else if (getAst1() instanceof IDAst) {
6164 				IDAst idAst = (IDAst) getAst1();
6165 				idAst.setArray(true);
6166 				tuples.deleteArray(idAst.offset, idAst.isGlobal);
6167 			} else {
6168 				throw new Error("Should never reach here : delete for " + getAst1());
6169 			}
6170 
6171 			popSourceLineNumber(tuples);
6172 			return 0;
6173 		}
6174 	}
6175 
6176 	private class BreakStatementAst extends AST {
6177 
6178 		@Override
6179 		public int populateTuples(AwkTuples tuples) {
6180 			pushSourceLineNumber(tuples);
6181 			AST breakable = searchFor(AstFlag.BREAKABLE);
6182 			if (breakable == null) {
6183 				throw new SemanticException("cannot break; not within a loop");
6184 			}
6185 			tuples.gotoAddress(breakable.breakAddress());
6186 			popSourceLineNumber(tuples);
6187 			return 0;
6188 		}
6189 	}
6190 
6191 	private class NextStatementAst extends AST {
6192 
6193 		@Override
6194 		public int populateTuples(AwkTuples tuples) {
6195 			pushSourceLineNumber(tuples);
6196 			AST nextable = searchFor(AstFlag.NEXTABLE);
6197 			if (nextable == null) {
6198 				// Inside a user-defined function: the calling rule cannot be
6199 				// known statically (the same function may be called from both
6200 				// an input rule and a special rule), so emit the runtime form,
6201 				// which unwinds the function calls and resumes the main input
6202 				// loop, or reports a fatal error for BEGIN, END, BEGINFILE,
6203 				// and ENDFILE callers.
6204 				tuples.execNext();
6205 			} else {
6206 				tuples.gotoAddress(nextable.nextAddress());
6207 			}
6208 			popSourceLineNumber(tuples);
6209 			return 0;
6210 		}
6211 	}
6212 
6213 	private class NextfileStatementAst extends AST {
6214 
6215 		@Override
6216 		public int populateTuples(AwkTuples tuples) {
6217 			pushSourceLineNumber(tuples);
6218 			AST nextable = searchFor(AstFlag.NEXTABLE);
6219 			if (nextable != null) {
6220 				// Direct use inside a rule: BEGIN, END, and ENDFILE reject
6221 				// nextfile at compile time, mirroring gawk's fatal errors.
6222 				// (Uses within user-defined functions are checked at runtime.)
6223 				AST pattern = nextable.getAst1();
6224 				if (pattern != null && (pattern.isBegin() || pattern.isEnd() || pattern.isEndFile())) {
6225 					String ruleName = pattern.isBegin() ? "BEGIN" : pattern.isEnd() ? "END" : "ENDFILE";
6226 					throw new SemanticException("`nextfile' cannot be called from a `" + ruleName + "' rule.");
6227 				}
6228 			}
6229 			tuples.execNextfile();
6230 			popSourceLineNumber(tuples);
6231 			return 0;
6232 		}
6233 	}
6234 
6235 	private final class ContinueStatementAst extends AST {
6236 
6237 		private ContinueStatementAst() {
6238 			super();
6239 		}
6240 
6241 		@Override
6242 		public int populateTuples(AwkTuples tuples) {
6243 			pushSourceLineNumber(tuples);
6244 			AST continueable = searchFor(AstFlag.CONTINUEABLE);
6245 			if (continueable == null) {
6246 				throw new SemanticException("cannot issue a continue; not within any loops");
6247 			}
6248 			tuples.gotoAddress(continueable.continueAddress());
6249 			popSourceLineNumber(tuples);
6250 			return 0;
6251 		}
6252 	}
6253 
6254 	// this was static...
6255 	// made non-static to throw a meaningful ParserException when necessary
6256 	private final class FunctionProxy implements Supplier<Address> {
6257 
6258 		private FunctionDefAst functionDefAst;
6259 		private String id;
6260 
6261 		private FunctionProxy(String id) {
6262 			this.id = id;
6263 		}
6264 
6265 		private void setFunctionDefinition(FunctionDefAst functionDef) {
6266 			if (functionDefAst != null) {
6267 				throw parserException("function " + functionDef + " already defined");
6268 			} else {
6269 				functionDefAst = functionDef;
6270 			}
6271 		}
6272 
6273 		private boolean isDefined() {
6274 			return functionDefAst != null;
6275 		}
6276 
6277 		@Override
6278 		public Address get() {
6279 			return functionDefAst.getAddress();
6280 		}
6281 
6282 		private String getFunctionName() {
6283 			return id;
6284 		}
6285 
6286 		private int getFunctionParamCount() {
6287 			return functionDefAst.paramCount();
6288 		}
6289 
6290 		@Override
6291 		public String toString() {
6292 			return super.toString() + " (" + id + ")";
6293 		}
6294 
6295 	}
6296 
6297 	/**
6298 	 * Adds {varName -&gt; offset} mappings to the tuples so that global variables
6299 	 * can be set by the interpreter while processing filename and name=value
6300 	 * entries from the command-line.
6301 	 * Also sends function names to the tuples, to provide the back end
6302 	 * with names to invalidate if name=value assignments are passed
6303 	 * in via the -v or ARGV arguments.
6304 	 *
6305 	 * @param tuples The tuples to add the mapping to.
6306 	 */
6307 	public void populateGlobalVariableNameToOffsetMappings(AwkTuples tuples) {
6308 		for (String varname : symbolTable.globalIds.keySet()) {
6309 			IDAst idAst = symbolTable.globalIds.get(varname);
6310 			// The last arg originally was ", idAst.isScalar", but this is not set true
6311 			// if the variable use is ambiguous. Therefore, assume it is a scalar
6312 			// if it's Token.NOT used as an array.
6313 			tuples.addGlobalVariableNameToOffsetMapping(varname, idAst.offset, idAst.isArray);
6314 		}
6315 		tuples.setFunctionNameSet(symbolTable.functionProxies.keySet());
6316 	}
6317 
6318 	private class AwkSymbolTableImpl {
6319 
6320 		int numGlobals() {
6321 			return globalIds.size();
6322 		}
6323 
6324 		// "constants"
6325 		private BeginAst beginAst = null;
6326 		private EndAst endAst = null;
6327 		private BeginFileAst beginFileAst = null;
6328 		private EndFileAst endFileAst = null;
6329 
6330 		// functions (proxies)
6331 		private Map<String, FunctionProxy> functionProxies = new HashMap<String, FunctionProxy>();
6332 		private Map<String, Tuple.IndirectFunctionTarget> cachedIndirectFunctionTargets;
6333 
6334 		// variable management
6335 		private Map<String, IDAst> globalIds = new HashMap<String, IDAst>();
6336 		private Map<String, Map<String, IDAst>> localIds = new HashMap<String, Map<String, IDAst>>();
6337 		private Map<String, Set<String>> functionParameters = new HashMap<String, Set<String>>();
6338 		private Set<String> ids = new HashSet<String>();
6339 
6340 		// current function definition for symbols
6341 		private String currentFunctionName = null;
6342 
6343 		// using set/clear rather than push/pop, it is impossible to define functions within functions
6344 		void setFunctionName(String functionName) {
6345 			this.currentFunctionName = functionName;
6346 		}
6347 
6348 		void clearFunctionName(String functionName) {
6349 			this.currentFunctionName = null;
6350 		}
6351 
6352 		AST addBEGIN() {
6353 			if (beginAst == null) {
6354 				beginAst = new BeginAst();
6355 			}
6356 			return beginAst;
6357 		}
6358 
6359 		AST addEND() {
6360 			if (endAst == null) {
6361 				endAst = new EndAst();
6362 			}
6363 			return endAst;
6364 		}
6365 
6366 		AST addBEGINFILE() {
6367 			if (beginFileAst == null) {
6368 				beginFileAst = new BeginFileAst();
6369 			}
6370 			return beginFileAst;
6371 		}
6372 
6373 		AST addENDFILE() {
6374 			if (endFileAst == null) {
6375 				endFileAst = new EndFileAst();
6376 			}
6377 			return endFileAst;
6378 		}
6379 
6380 		/**
6381 		 * Returns whether the script references the named global variable,
6382 		 * without creating a symbol for it.
6383 		 */
6384 		private boolean isGlobalReferenced(String id) {
6385 			IDAst idAst = globalIds.get(id);
6386 			return idAst != null && idAst.isReferenced();
6387 		}
6388 
6389 		private IDAst getID(String id) {
6390 			id = resolveVariableIdentifier(id);
6391 
6392 			Map<String, IDAst> map;
6393 			if (currentFunctionName == null) {
6394 				map = globalIds;
6395 			} else {
6396 				Set<String> set = functionParameters.get(currentFunctionName);
6397 				// we need "set != null && ..." here because if function
6398 				// is defined with no args (i.e., function f() ...),
6399 				// then set is null
6400 				if (set != null && set.contains(id)) {
6401 					map = localIds.get(currentFunctionName);
6402 					if (map == null) {
6403 						map = new HashMap<String, IDAst>();
6404 						localIds.put(currentFunctionName, map);
6405 					}
6406 				} else {
6407 					map = globalIds;
6408 				}
6409 			}
6410 			if (map == globalIds) {
6411 				// Only global variables share the namespace with function names.
6412 				// Formal parameters may legitimately have the same name as an
6413 				// unrelated function.
6414 				if (functionProxies.get(id) != null) {
6415 					throw parserException("cannot use " + id + " as a variable; it is a function");
6416 				}
6417 				ids.add(id);
6418 			}
6419 			IDAst idAst = map.get(id);
6420 			if (idAst == null) {
6421 				idAst = new IDAst(id, map == globalIds);
6422 				idAst.offset = map.size();
6423 				if (map == globalIds && !posix && ("SYMTAB".equals(id) || "FUNCTAB".equals(id))) {
6424 					// the runtime-provided meta tables are array-only, as in
6425 					// gawk: using them as scalars must fail like any array
6426 					idAst.setArray(true);
6427 				}
6428 				map.put(id, idAst);
6429 			}
6430 			return idAst;
6431 		}
6432 
6433 		private String resolveVariableIdentifier(String id) {
6434 			if (currentFunctionName != null) {
6435 				Set<String> parameters = functionParameters.get(currentFunctionName);
6436 				if (parameters != null && parameters.contains(id)) {
6437 					return id;
6438 				}
6439 			}
6440 			return qualifyGlobalIdentifier(id);
6441 		}
6442 
6443 		AST addID(String id) throws ParserException {
6444 			IDAst retVal = getID(id);
6445 			retVal.markReferenced();
6446 			/// ***
6447                         /// We really don't know if the evaluation is for an array or for a scalar
6448                         /// here, because we can use an array as a function parameter (passed by reference).
6449 			/// ***
6450 			// if (retVal.isArray)
6451 			// throw parserException("Cannot use "+retVal+" as a scalar.");
6452 			// retVal.isScalar = true;
6453 			return retVal;
6454 		}
6455 
6456 		int addFunctionParameter(String functionName, String id) {
6457 			int namespaceSeparator = functionName.indexOf("::");
6458 			String unqualifiedFunctionName = namespaceSeparator < 0 ?
6459 					functionName :
6460 					functionName.substring(namespaceSeparator + 2);
6461 			if (unqualifiedFunctionName.equals(id)) {
6462 				throw parserException("cannot use " + id + " as a parameter; it is the function name");
6463 			}
6464 			Set<String> set = functionParameters.get(functionName);
6465 			if (set == null) {
6466 				set = new HashSet<String>();
6467 				functionParameters.put(functionName, set);
6468 			}
6469 			if (set.contains(id)) {
6470 				throw parserException("multiply defined parameter " + id + " in function " + functionName);
6471 			}
6472 			int retval = set.size();
6473 			set.add(id);
6474 			Map<String, IDAst> map = localIds.get(functionName);
6475 			if (map == null) {
6476 				map = new HashMap<String, IDAst>();
6477 				localIds.put(functionName, map);
6478 			}
6479 			IDAst idAst = map.get(id);
6480 			if (idAst == null) {
6481 				idAst = new IDAst(id, map == globalIds);
6482 				idAst.offset = map.size();
6483 				map.put(id, idAst);
6484 			}
6485 
6486 			return retval;
6487 		}
6488 
6489 		IDAst getFunctionParameterIDAST(String functionName, String fIdString) {
6490 			return localIds.get(functionName).get(fIdString);
6491 		}
6492 
6493 		AST addArrayID(String id) throws ParserException {
6494 			IDAst retVal = getID(id);
6495 			retVal.markReferenced();
6496 			retVal.setArray(true);
6497 			return retVal;
6498 		}
6499 
6500 		AST addFunctionDef(String functionName, AST paramList, AST block) {
6501 			if (ids.contains(functionName)) {
6502 				throw parserException("cannot use " + functionName + " as a function; it is a variable");
6503 			}
6504 			FunctionProxy functionProxy = functionProxies.get(functionName);
6505 			if (functionProxy == null) {
6506 				functionProxy = new FunctionProxy(functionName);
6507 				functionProxies.put(functionName, functionProxy);
6508 			}
6509 			FunctionDefAst functionDef = new FunctionDefAst(functionName, paramList, block);
6510 			functionProxy.setFunctionDefinition(functionDef);
6511 			return functionDef;
6512 		}
6513 
6514 		AST addFunctionCall(String id, AST paramList) {
6515 			id = qualifyGlobalIdentifier(id);
6516 			FunctionProxy functionProxy = functionProxies.get(id);
6517 			if (functionProxy == null) {
6518 				functionProxy = new FunctionProxy(id);
6519 				functionProxies.put(id, functionProxy);
6520 			}
6521 			return new FunctionCallAst(functionProxy, paramList);
6522 		}
6523 
6524 		Map<String, Tuple.IndirectFunctionTarget> indirectFunctionTargets() {
6525 			if (cachedIndirectFunctionTargets != null) {
6526 				return cachedIndirectFunctionTargets;
6527 			}
6528 			Map<String, Tuple.IndirectFunctionTarget> targets = new HashMap<String, Tuple.IndirectFunctionTarget>();
6529 			for (Map.Entry<String, FunctionProxy> entry : functionProxies.entrySet()) {
6530 				FunctionProxy proxy = entry.getValue();
6531 				if (proxy.isDefined()) {
6532 					targets.put(
6533 							entry.getKey(),
6534 							new Tuple.IndirectFunctionTarget(
6535 									proxy,
6536 									proxy.getFunctionParamCount(),
6537 									collectArrayParameterIndexes(proxy.functionDefAst)));
6538 				}
6539 			}
6540 			cachedIndirectFunctionTargets = Collections.unmodifiableMap(targets);
6541 			return cachedIndirectFunctionTargets;
6542 		}
6543 
6544 		AST addArrayReference(String id, AST idxAst, int lineNo) throws ParserException {
6545 			return new ArrayReferenceAst(lineNo, addArrayID(id), idxAst);
6546 		}
6547 
6548 		// constants are no longer cached/hashed so that individual ASTs
6549 		// can report accurate line numbers upon errors
6550 
6551 		AST addINTEGER(String integer) {
6552 			try {
6553 				return new IntegerAst(Long.parseLong(integer));
6554 			} catch (NumberFormatException beyondLongRange) {
6555 				// A literal too large for a 64-bit integer is a floating-point
6556 				// constant, as in gawk, rather than a parse error.
6557 				return new DoubleAst(Double.parseDouble(integer));
6558 			}
6559 		}
6560 
6561 		AST addDOUBLE(String dbl) {
6562 			return new DoubleAst(Double.valueOf(dbl));
6563 		}
6564 
6565 		AST addSTRING(String str) {
6566 			return new StringAst(str);
6567 		}
6568 
6569 		AST addREGEXP(String localRegexp) {
6570 			return new RegexpAst(localRegexp, false);
6571 		}
6572 
6573 		AST addTYPED_REGEXP(String localRegexp) {
6574 			return new RegexpAst(localRegexp, true);
6575 		}
6576 	}
6577 
6578 	private ParserException parserException(String msg) {
6579 		return new ParserException(msg, currentScriptSource.getDescription(), reader.getLineNumber());
6580 	}
6581 }