1 package io.jawk;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 import java.io.ByteArrayInputStream;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.OutputStream;
29 import java.io.PrintStream;
30 import java.io.Reader;
31 import java.io.StringReader;
32 import java.nio.charset.StandardCharsets;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.LinkedHashMap;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Objects;
41 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
42 import io.jawk.backend.AVM;
43 import io.jawk.ext.ExtensionFunction;
44 import io.jawk.ext.ExtensionRegistry;
45 import io.jawk.ext.JawkExtension;
46 import io.jawk.frontend.AwkParser;
47 import io.jawk.frontend.AstNode;
48 import io.jawk.jrt.AppendableAwkSink;
49 import io.jawk.jrt.AwkSink;
50 import io.jawk.jrt.InputSource;
51 import io.jawk.jrt.OutputStreamAwkSink;
52 import io.jawk.jrt.StreamInputSource;
53 import io.jawk.util.AwkSettings;
54 import io.jawk.util.ScriptSource;
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84 public class Awk {
85
86
87 public static final String DEFAULT_FS = " ";
88
89
90 public static final String DEFAULT_RS = "\n";
91
92
93 public static final String DEFAULT_OFS = " ";
94
95
96 public static final String DEFAULT_ORS = "\n";
97
98
99 public static final String DEFAULT_CONVFMT = "%.6g";
100
101
102 public static final String DEFAULT_OFMT = "%.6g";
103
104
105 public static final String DEFAULT_SUBSEP = String.valueOf((char) 28);
106
107 private final Map<String, ExtensionFunction> extensionFunctions;
108
109 private final Map<String, JawkExtension> extensionInstances;
110
111
112
113
114 private final AwkSettings settings;
115
116
117
118
119 private AstNode lastAst;
120
121
122
123
124 public Awk() {
125 this(new AwkSettings());
126 }
127
128
129
130
131
132
133 public Awk(AwkSettings settings) {
134 this(ExtensionSetup.EMPTY, settings);
135 }
136
137
138
139
140
141
142 public Awk(Collection<? extends JawkExtension> extensions) {
143 this(createExtensionSetup(extensions));
144 }
145
146
147
148
149
150
151
152
153 public Awk(Collection<? extends JawkExtension> extensions, AwkSettings settings) {
154 this(createExtensionSetup(extensions), settings);
155 }
156
157
158
159
160
161
162 @SafeVarargs
163 public Awk(JawkExtension... extensions) {
164 this(createExtensionSetup(Arrays.asList(extensions)));
165 }
166
167 protected Awk(ExtensionSetup setup) {
168 this(setup, new AwkSettings());
169 }
170
171 protected Awk(ExtensionSetup setup, AwkSettings settings) {
172 this.extensionFunctions = setup.functions;
173 this.extensionInstances = setup.instances;
174 this.settings = Objects.requireNonNull(settings, "settings");
175 }
176
177 protected Map<String, ExtensionFunction> getExtensionFunctions() {
178 return extensionFunctions;
179 }
180
181 protected Map<String, JawkExtension> getExtensionInstances() {
182 return extensionInstances;
183 }
184
185
186
187
188
189
190 @SuppressFBWarnings("EI_EXPOSE_REP")
191 public AwkSettings getSettings() {
192 return settings;
193 }
194
195 static Map<String, ExtensionFunction> createExtensionFunctionMap(Collection<? extends JawkExtension> extensions) {
196 return createExtensionSetup(extensions).functions;
197 }
198
199 static Map<String, JawkExtension> createExtensionInstanceMap(Collection<? extends JawkExtension> extensions) {
200 return createExtensionSetup(extensions).instances;
201 }
202
203 static Map<String, ExtensionFunction> createExtensionFunctionMap(JawkExtension... extensions) {
204 if (extensions == null || extensions.length == 0) {
205 return ExtensionSetup.EMPTY.functions;
206 }
207 return createExtensionFunctionMap(Arrays.asList(extensions));
208 }
209
210 static Map<String, JawkExtension> createExtensionInstanceMap(JawkExtension... extensions) {
211 if (extensions == null || extensions.length == 0) {
212 return ExtensionSetup.EMPTY.instances;
213 }
214 return createExtensionInstanceMap(Arrays.asList(extensions));
215 }
216
217 private static ExtensionSetup createExtensionSetup(Collection<? extends JawkExtension> extensions) {
218 if (extensions == null || extensions.isEmpty()) {
219 return ExtensionSetup.EMPTY;
220 }
221 Map<String, ExtensionFunction> keywordMap = new LinkedHashMap<String, ExtensionFunction>();
222 Map<String, JawkExtension> instanceMap = new LinkedHashMap<String, JawkExtension>();
223 for (JawkExtension extension : extensions) {
224 if (extension == null) {
225 throw new IllegalArgumentException("Extension instance must not be null");
226 }
227 String className = extension.getClass().getName();
228 JawkExtension previousInstance = instanceMap.putIfAbsent(className, extension);
229 if (previousInstance != null) {
230 throw new IllegalArgumentException(
231 "Extension class '" + className + "' was provided multiple times");
232 }
233 for (Map.Entry<String, ExtensionFunction> entry : extension.getExtensionFunctions().entrySet()) {
234 String keyword = entry.getKey();
235 ExtensionFunction previous = keywordMap.putIfAbsent(keyword, entry.getValue());
236 if (previous != null) {
237 throw new IllegalArgumentException(
238 "Keyword '" + keyword + "' already provided by another extension");
239 }
240 }
241 }
242 return new ExtensionSetup(
243 Collections.unmodifiableMap(keywordMap),
244 Collections.unmodifiableMap(instanceMap));
245 }
246
247 private static final class ExtensionSetup {
248
249 private static final ExtensionSetup EMPTY = new ExtensionSetup(
250 Collections.<String, ExtensionFunction>emptyMap(),
251 Collections.<String, JawkExtension>emptyMap());
252
253 private final Map<String, ExtensionFunction> functions;
254 private final Map<String, JawkExtension> instances;
255
256 private ExtensionSetup(Map<String, ExtensionFunction> functionsParam,
257 Map<String, JawkExtension> instancesParam) {
258 this.functions = functionsParam;
259 this.instances = instancesParam;
260 }
261 }
262
263
264
265
266
267
268 @SuppressFBWarnings("EI_EXPOSE_REP")
269 public AstNode getLastAst() {
270 return lastAst;
271 }
272
273
274
275
276
277
278 @SuppressWarnings("deprecation")
279 @Override
280 protected final void finalize() { }
281
282
283
284
285
286
287
288
289 public AwkProgram compile(String script) throws IOException {
290 return compile(script, false);
291 }
292
293
294
295
296
297
298
299
300 public AwkProgram compile(Reader script) throws IOException {
301 return compile(script, false);
302 }
303
304
305
306
307
308
309 public AVM createAvm() {
310 return createAvm(this.settings);
311 }
312
313
314
315
316
317
318
319
320 public AVM createAvm(boolean profilingEnabled) {
321 return createAvm(this.settings, profilingEnabled);
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339 public AwkRunBuilder script(AwkProgram program) {
340 return new AwkRunBuilder(Objects.requireNonNull(program, "program"));
341 }
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358 public AwkRunBuilder script(String scriptText) {
359 return new AwkRunBuilder().script(Objects.requireNonNull(scriptText, "script"));
360 }
361
362
363
364
365
366
367
368
369 public Object eval(AwkExpression expression) throws IOException {
370 AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression");
371 try (AVM activeEvalAvm = createAvm(settings)) {
372 return activeEvalAvm.eval(compiledExpression, new SingleRecordInputSource(null));
373 }
374 }
375
376
377
378
379
380
381
382
383
384
385 public Object eval(AwkExpression expression, String input) throws IOException {
386 AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression");
387 try (AVM activeEvalAvm = createAvm(settings)) {
388 return activeEvalAvm.eval(compiledExpression, new SingleRecordInputSource(input));
389 }
390 }
391
392
393
394
395
396
397
398
399
400
401 public Object eval(AwkExpression expression, InputSource source) throws IOException {
402 AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression");
403 InputSource resolvedSource = Objects.requireNonNull(source, "source");
404 try (AVM activeEvalAvm = createAvm(settings)) {
405 return activeEvalAvm.eval(compiledExpression, resolvedSource);
406 }
407 }
408
409
410
411
412
413
414
415
416
417 AwkProgram compile(String script, boolean disableOptimizeParam) throws IOException {
418 ScriptSource source = new ScriptSource(
419 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
420 new StringReader(script));
421 return compile(Collections.singletonList(source), disableOptimizeParam);
422 }
423
424
425
426
427
428
429
430
431
432 AwkProgram compile(Reader script, boolean disableOptimizeParam) throws IOException {
433 ScriptSource source = new ScriptSource(
434 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
435 script);
436 return compile(Collections.singletonList(source), disableOptimizeParam);
437 }
438
439
440
441
442
443
444
445
446
447
448 public AwkProgram compile(List<ScriptSource> scripts)
449 throws IOException {
450 return compile(scripts, false);
451 }
452
453
454
455
456
457
458
459
460
461
462
463 public AwkProgram compile(List<ScriptSource> scripts, boolean disableOptimizeParam)
464 throws IOException {
465 return compileProgram(scripts, disableOptimizeParam, new AwkProgram());
466 }
467
468
469
470
471
472
473
474
475
476
477
478 protected final <T extends AwkProgram> T compileProgram(
479 List<ScriptSource> scripts,
480 boolean disableOptimizeParam,
481 T tuples)
482 throws IOException {
483 lastAst = null;
484 if (!scripts.isEmpty()) {
485
486 AwkParser parser = new AwkParser(this.extensionFunctions, settings.isAllowArraysOfArrays());
487 AstNode ast = parser.parse(scripts);
488 lastAst = ast;
489 if (ast != null) {
490
491 ast.semanticAnalysis();
492 ast.semanticAnalysis();
493
494 ast.populateTuples(tuples);
495
496 tuples.postProcess();
497 if (!disableOptimizeParam) {
498 tuples.optimize();
499 }
500
501 parser.populateGlobalVariableNameToOffsetMappings(tuples);
502 }
503 }
504 tuples.freezeMetadata();
505
506 return tuples;
507 }
508
509
510
511
512
513
514
515
516 public AwkExpression compileExpression(String expression) throws IOException {
517 return compileExpression(expression, false);
518 }
519
520
521
522
523
524
525
526
527
528 public AwkExpression compileExpression(String expression, boolean disableOptimizeParam) throws IOException {
529 return compileExpression(expression, disableOptimizeParam, new AwkExpression());
530 }
531
532
533
534
535
536
537
538
539
540
541
542 protected final <T extends AwkExpression> T compileExpression(
543 String expression,
544 boolean disableOptimizeParam,
545 T tuples)
546 throws IOException {
547
548 ScriptSource expressionSource = new ScriptSource(
549 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
550 new StringReader(expression));
551
552
553 AwkParser parser = new AwkParser(this.extensionFunctions, settings.isAllowArraysOfArrays());
554 AstNode ast = parser.parseExpression(expressionSource);
555
556
557
558 if (ast != null) {
559
560 ast.semanticAnalysis();
561
562 ast.semanticAnalysis();
563
564 ast.populateTuples(tuples);
565
566 tuples.postProcess();
567 if (!disableOptimizeParam) {
568 tuples.optimize();
569 }
570
571
572 parser.populateGlobalVariableNameToOffsetMappings(tuples);
573 }
574 tuples.freezeMetadata();
575
576 return tuples;
577 }
578
579
580
581
582
583
584
585
586
587 public Object eval(String expression) throws IOException {
588 return eval(compileExpression(expression));
589 }
590
591
592
593
594
595
596
597
598
599
600 public Object eval(String expression, String input) throws IOException {
601 return eval(compileExpression(expression), input);
602 }
603
604
605
606
607
608
609
610
611
612
613 public Object eval(String expression, InputSource source) throws IOException {
614 return eval(compileExpression(expression), source);
615 }
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635 public AVM prepareEval(String input) throws IOException {
636 String resolvedInput = Objects.requireNonNull(input, "input");
637 AVM evalAvm = createAvm(settings);
638 try {
639 evalAvm.prepareForEval(resolvedInput);
640 return evalAvm;
641 } catch (IOException | RuntimeException e) {
642 try {
643 evalAvm.close();
644 } catch (IOException closeException) {
645 e.addSuppressed(closeException);
646 }
647 throw e;
648 }
649 }
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669 public AVM prepareEval(InputSource source) throws IOException {
670 InputSource resolvedSource = Objects.requireNonNull(source, "source");
671 AVM evalAvm = createAvm(settings);
672 try {
673 if (!evalAvm.prepareForEval(resolvedSource)) {
674 throw new IOException("No record available from source.");
675 }
676 return evalAvm;
677 } catch (IOException | RuntimeException e) {
678 try {
679 evalAvm.close();
680 } catch (IOException closeException) {
681 e.addSuppressed(closeException);
682 }
683 throw e;
684 }
685 }
686
687
688
689
690
691
692
693 protected AVM createAvm(AwkSettings settingsParam) {
694 return createAvm(settingsParam, false);
695 }
696
697
698
699
700
701
702
703
704
705 protected AVM createAvm(AwkSettings settingsParam, boolean profilingEnabled) {
706 return new AVM(settingsParam, this.extensionInstances, profilingEnabled);
707 }
708
709
710
711
712 private static InputStream toInputStream(String input) {
713 if (input == null) {
714 return new ByteArrayInputStream(new byte[0]);
715 }
716 return new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8));
717 }
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741 public final class AwkRunBuilder {
742
743 private AwkProgram compiledProgram;
744 private List<String> scripts;
745 private InputStream inputStream;
746 private InputSource inputSource;
747 private List<String> arguments;
748 private Map<String, Object> variableOverrides;
749 private PrintStream errorStream;
750
751 AwkRunBuilder() {}
752
753 AwkRunBuilder(AwkProgram program) {
754 this.compiledProgram = program;
755 }
756
757
758
759
760
761
762
763
764
765
766 public AwkRunBuilder script(String scriptText) {
767 if (compiledProgram != null) {
768 throw new IllegalStateException("Cannot add scripts when a precompiled program is set");
769 }
770 if (scripts == null) {
771 scripts = new ArrayList<String>();
772 }
773 scripts.add(Objects.requireNonNull(scriptText, "script"));
774 return this;
775 }
776
777
778
779
780
781
782
783 public AwkRunBuilder input(String input) {
784 this.inputStream = toInputStream(input);
785 return this;
786 }
787
788
789
790
791
792
793
794 public AwkRunBuilder input(InputStream input) {
795 this.inputStream = input;
796 return this;
797 }
798
799
800
801
802
803
804
805 public AwkRunBuilder input(InputSource source) {
806 this.inputSource = source;
807 return this;
808 }
809
810
811
812
813
814
815
816 @SuppressFBWarnings("EI_EXPOSE_REP2")
817 public AwkRunBuilder arguments(List<String> args) {
818 this.arguments = args;
819 return this;
820 }
821
822
823
824
825
826
827
828 public AwkRunBuilder arguments(String... args) {
829 this.arguments = Arrays.asList(args);
830 return this;
831 }
832
833
834
835
836
837
838
839 public AwkRunBuilder argument(String arg) {
840 if (this.arguments == null) {
841 this.arguments = new ArrayList<String>();
842 }
843 this.arguments.add(Objects.requireNonNull(arg, "arg"));
844 return this;
845 }
846
847
848
849
850
851
852
853
854
855
856
857
858 public AwkRunBuilder errorStream(PrintStream stream) {
859 this.errorStream = Objects.requireNonNull(stream, "errorStream");
860 return this;
861 }
862
863
864
865
866
867
868
869
870 @SuppressFBWarnings("EI_EXPOSE_REP2")
871 public AwkRunBuilder variables(Map<String, Object> overrides) {
872 this.variableOverrides = overrides;
873 return this;
874 }
875
876
877
878
879
880
881
882
883 public AwkRunBuilder variable(String name, Object value) {
884 if (this.variableOverrides == null) {
885 this.variableOverrides = new LinkedHashMap<String, Object>();
886 }
887 this.variableOverrides
888 .put(
889 Objects.requireNonNull(name, "name"),
890 value);
891 return this;
892 }
893
894
895
896
897
898
899
900
901 public String execute() throws IOException, ExitException {
902 StringBuilder output = new StringBuilder();
903 doExecute(new AppendableAwkSink(output, settings.getLocale()));
904 return output.toString();
905 }
906
907
908
909
910
911
912
913
914 public void execute(AwkSink sink) throws IOException, ExitException {
915 doExecute(Objects.requireNonNull(sink, "sink"));
916 }
917
918
919
920
921
922
923
924
925 public void execute(PrintStream out) throws IOException, ExitException {
926 Objects.requireNonNull(out, "out");
927 doExecute(new OutputStreamAwkSink(out, settings.getLocale()));
928 }
929
930
931
932
933
934
935
936
937 public void execute(OutputStream out) throws IOException, ExitException {
938 doExecute(new OutputStreamAwkSink(toPrintStream(out), settings.getLocale()));
939 }
940
941
942
943
944
945
946
947
948
949 public void execute(Appendable appendable) throws IOException, ExitException {
950 doExecute(
951 new AppendableAwkSink(
952 Objects.requireNonNull(appendable, "appendable"),
953 settings.getLocale()));
954 }
955
956 private void doExecute(AwkSink sink) throws IOException, ExitException {
957 AwkProgram program = resolveProgram();
958 List<String> resolvedArguments = arguments == null ? Collections.<String>emptyList() : arguments;
959 try (AVM avm = createAvm(settings)) {
960 avm.setAwkSink(sink);
961 avm.setErrorStream(errorStream != null ? errorStream : sink.getPrintStream());
962 try {
963 InputSource resolvedSource;
964 if (inputSource != null) {
965 resolvedSource = inputSource;
966 } else {
967 InputStream in = inputStream != null ? inputStream : new ByteArrayInputStream(new byte[0]);
968 resolvedSource = new StreamInputSource(in, avm, avm.getJrt());
969 }
970 avm.execute(program, resolvedSource, resolvedArguments, variableOverrides);
971 } catch (ExitException e) {
972 if (e.getCode() != 0) {
973 throw e;
974 }
975 } finally {
976 sink.flush();
977 }
978 }
979 }
980
981 private AwkProgram resolveProgram() throws IOException {
982 if (compiledProgram != null) {
983 return compiledProgram;
984 }
985 if (scripts == null || scripts.isEmpty()) {
986 throw new IllegalStateException("No script or program specified");
987 }
988 if (scripts.size() == 1) {
989 return compile(scripts.get(0));
990 }
991 List<ScriptSource> sources = new ArrayList<ScriptSource>(scripts.size());
992 for (int i = 0; i < scripts.size(); i++) {
993 sources
994 .add(
995 new ScriptSource(
996 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
997 new StringReader(scripts.get(i))));
998 }
999 return compile(sources);
1000 }
1001 }
1002
1003 private static PrintStream toPrintStream(OutputStream out) {
1004 Objects.requireNonNull(out, "outputStream");
1005 if (out instanceof PrintStream) {
1006 return (PrintStream) out;
1007 }
1008 try {
1009 return new PrintStream(out, false, "UTF-8");
1010 } catch (java.io.UnsupportedEncodingException e) {
1011 throw new IllegalStateException(e);
1012 }
1013 }
1014
1015
1016
1017
1018
1019
1020
1021 public static Map<String, JawkExtension> listAvailableExtensions() {
1022 return ExtensionRegistry.listExtensions();
1023 }
1024
1025 private static final class SingleRecordInputSource implements InputSource {
1026
1027 private final String record;
1028
1029 private boolean consumed;
1030
1031 private SingleRecordInputSource(String record) {
1032 this.record = record;
1033 }
1034
1035 @Override
1036 public boolean nextRecord() {
1037 if (consumed || record == null) {
1038 return false;
1039 }
1040 consumed = true;
1041 return true;
1042 }
1043
1044 @Override
1045 public String getRecordText() {
1046 return consumed ? record : null;
1047 }
1048
1049 @Override
1050 public List<String> getFields() {
1051 return null;
1052 }
1053
1054 @Override
1055 public boolean isFromFilenameList() {
1056 return false;
1057 }
1058 }
1059
1060 }