1 package io.jawk;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 import java.io.ByteArrayInputStream;
26 import java.io.IOException;
27 import java.io.InputStream;
28 import java.io.OutputStream;
29 import java.io.PrintStream;
30 import java.io.Reader;
31 import java.io.StringReader;
32 import java.nio.charset.StandardCharsets;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.LinkedHashMap;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Objects;
41 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
42 import io.jawk.backend.AVM;
43 import io.jawk.ext.ExtensionFunction;
44 import io.jawk.ext.ExtensionRegistry;
45 import io.jawk.ext.GawkExtension;
46 import io.jawk.ext.JawkExtension;
47 import io.jawk.frontend.AwkParser;
48 import io.jawk.frontend.AstNode;
49 import io.jawk.jrt.AppendableAwkSink;
50 import io.jawk.jrt.AwkSink;
51 import io.jawk.jrt.InputSource;
52 import io.jawk.jrt.OutputStreamAwkSink;
53 import io.jawk.jrt.StreamInputSource;
54 import io.jawk.util.AwkSettings;
55 import io.jawk.util.ScriptSource;
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85 public class Awk {
86
87
88 public static final String DEFAULT_FS = " ";
89
90
91 public static final String DEFAULT_RS = "\n";
92
93
94 public static final String DEFAULT_OFS = " ";
95
96
97 public static final String DEFAULT_ORS = "\n";
98
99
100 public static final String DEFAULT_CONVFMT = "%.6g";
101
102
103 public static final String DEFAULT_OFMT = "%.6g";
104
105
106 public static final String DEFAULT_SUBSEP = String.valueOf((char) 28);
107
108 private final Map<String, ExtensionFunction> extensionFunctions;
109
110 private final Map<String, JawkExtension> extensionInstances;
111
112
113
114
115 private final AwkSettings settings;
116
117
118
119
120 private AstNode lastAst;
121
122
123
124
125 public Awk() {
126 this(new AwkSettings());
127 }
128
129
130
131
132
133
134 public Awk(AwkSettings settings) {
135 this(ExtensionSetup.createDefault(), settings);
136 }
137
138
139
140
141
142
143 public Awk(Collection<? extends JawkExtension> extensions) {
144 this(createExtensionSetup(extensions));
145 }
146
147
148
149
150
151
152
153
154 public Awk(Collection<? extends JawkExtension> extensions, AwkSettings settings) {
155 this(createExtensionSetup(extensions), settings);
156 }
157
158
159
160
161
162
163 @SafeVarargs
164 public Awk(JawkExtension... extensions) {
165 this(createExtensionSetup(Arrays.asList(extensions)));
166 }
167
168 protected Awk(ExtensionSetup setup) {
169 this(setup, new AwkSettings());
170 }
171
172 protected Awk(ExtensionSetup setup, AwkSettings settings) {
173 this.extensionFunctions = setup.functions;
174 this.extensionInstances = setup.instances;
175 this.settings = Objects.requireNonNull(settings, "settings");
176 }
177
178 protected Map<String, ExtensionFunction> getExtensionFunctions() {
179 return extensionFunctions;
180 }
181
182 protected Map<String, JawkExtension> getExtensionInstances() {
183 return extensionInstances;
184 }
185
186
187
188
189
190
191 @SuppressFBWarnings("EI_EXPOSE_REP")
192 public AwkSettings getSettings() {
193 return settings;
194 }
195
196 static Map<String, ExtensionFunction> createExtensionFunctionMap(Collection<? extends JawkExtension> extensions) {
197 return createExtensionSetup(extensions).functions;
198 }
199
200 static Map<String, JawkExtension> createExtensionInstanceMap(Collection<? extends JawkExtension> extensions) {
201 return createExtensionSetup(extensions).instances;
202 }
203
204 static Map<String, ExtensionFunction> createExtensionFunctionMap(JawkExtension... extensions) {
205 return createExtensionFunctionMap(
206 extensions == null ? Collections.<JawkExtension>emptyList() : Arrays.asList(extensions));
207 }
208
209 static Map<String, JawkExtension> createExtensionInstanceMap(JawkExtension... extensions) {
210 return createExtensionInstanceMap(
211 extensions == null ? Collections.<JawkExtension>emptyList() : Arrays.asList(extensions));
212 }
213
214
215
216
217
218
219
220 private static ExtensionSetup createExtensionSetup(Collection<? extends JawkExtension> extensions) {
221 if (extensions == null || extensions.isEmpty()) {
222 return ExtensionSetup.EMPTY;
223 }
224 Map<String, ExtensionFunction> keywordMap = new LinkedHashMap<String, ExtensionFunction>();
225 Map<String, JawkExtension> instanceMap = new LinkedHashMap<String, JawkExtension>();
226 for (JawkExtension extension : extensions) {
227 if (extension == null) {
228 throw new IllegalArgumentException("Extension instance must not be null");
229 }
230 String className = extension.getClass().getName();
231 JawkExtension previousInstance = instanceMap.putIfAbsent(className, extension);
232 if (previousInstance != null) {
233 throw new IllegalArgumentException(
234 "Extension class '" + className + "' was provided multiple times");
235 }
236 for (Map.Entry<String, ExtensionFunction> entry : extension.getExtensionFunctions().entrySet()) {
237 String keyword = entry.getKey();
238 ExtensionFunction previous = keywordMap.putIfAbsent(keyword, entry.getValue());
239 if (previous != null) {
240 throw new IllegalArgumentException(
241 "Keyword '" + keyword + "' already provided by another extension");
242 }
243 }
244 }
245 return new ExtensionSetup(
246 Collections.unmodifiableMap(keywordMap),
247 Collections.unmodifiableMap(instanceMap));
248 }
249
250 private static final class ExtensionSetup {
251
252 private static final ExtensionSetup EMPTY = new ExtensionSetup(
253 Collections.<String, ExtensionFunction>emptyMap(),
254 Collections.<String, JawkExtension>emptyMap());
255
256
257
258
259
260
261
262 private static ExtensionSetup createDefault() {
263 return createExtensionSetup(Collections.singletonList(new GawkExtension()));
264 }
265
266 private final Map<String, ExtensionFunction> functions;
267 private final Map<String, JawkExtension> instances;
268
269 private ExtensionSetup(Map<String, ExtensionFunction> functionsParam,
270 Map<String, JawkExtension> instancesParam) {
271 this.functions = functionsParam;
272 this.instances = instancesParam;
273 }
274 }
275
276
277
278
279
280
281 @SuppressFBWarnings("EI_EXPOSE_REP")
282 public AstNode getLastAst() {
283 return lastAst;
284 }
285
286
287
288
289
290
291 @SuppressWarnings("deprecation")
292 @Override
293 protected final void finalize() { }
294
295
296
297
298
299
300
301
302 public AwkProgram compile(String script) throws IOException {
303 return compile(script, false);
304 }
305
306
307
308
309
310
311
312
313 public AwkProgram compile(Reader script) throws IOException {
314 return compile(script, false);
315 }
316
317
318
319
320
321
322 public AVM createAvm() {
323 return createAvm(this.settings);
324 }
325
326
327
328
329
330
331
332
333 public AVM createAvm(boolean profilingEnabled) {
334 return createAvm(this.settings, profilingEnabled);
335 }
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352 public AwkRunBuilder script(AwkProgram program) {
353 return new AwkRunBuilder(Objects.requireNonNull(program, "program"));
354 }
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371 public AwkRunBuilder script(String scriptText) {
372 return new AwkRunBuilder().script(Objects.requireNonNull(scriptText, "script"));
373 }
374
375
376
377
378
379
380
381
382 public Object eval(AwkExpression expression) throws IOException {
383 AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression");
384 try (AVM activeEvalAvm = createAvm(settings)) {
385 return activeEvalAvm.eval(compiledExpression, new SingleRecordInputSource(null));
386 }
387 }
388
389
390
391
392
393
394
395
396
397
398 public Object eval(AwkExpression expression, String input) throws IOException {
399 AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression");
400 try (AVM activeEvalAvm = createAvm(settings)) {
401 return activeEvalAvm.eval(compiledExpression, new SingleRecordInputSource(input));
402 }
403 }
404
405
406
407
408
409
410
411
412
413
414 public Object eval(AwkExpression expression, InputSource source) throws IOException {
415 AwkExpression compiledExpression = Objects.requireNonNull(expression, "expression");
416 InputSource resolvedSource = Objects.requireNonNull(source, "source");
417 try (AVM activeEvalAvm = createAvm(settings)) {
418 return activeEvalAvm.eval(compiledExpression, resolvedSource);
419 }
420 }
421
422
423
424
425
426
427
428
429
430 AwkProgram compile(String script, boolean disableOptimizeParam) throws IOException {
431 ScriptSource source = new ScriptSource(
432 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
433 new StringReader(script));
434 return compile(Collections.singletonList(source), disableOptimizeParam);
435 }
436
437
438
439
440
441
442
443
444
445 AwkProgram compile(Reader script, boolean disableOptimizeParam) throws IOException {
446 ScriptSource source = new ScriptSource(
447 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
448 script);
449 return compile(Collections.singletonList(source), disableOptimizeParam);
450 }
451
452
453
454
455
456
457
458
459
460
461 public AwkProgram compile(List<ScriptSource> scripts)
462 throws IOException {
463 return compile(scripts, false);
464 }
465
466
467
468
469
470
471
472
473
474
475
476 public AwkProgram compile(List<ScriptSource> scripts, boolean disableOptimizeParam)
477 throws IOException {
478 return compileProgram(scripts, disableOptimizeParam, new AwkProgram());
479 }
480
481
482
483
484
485
486
487
488
489
490
491 protected final <T extends AwkProgram> T compileProgram(
492 List<ScriptSource> scripts,
493 boolean disableOptimizeParam,
494 T tuples)
495 throws IOException {
496 lastAst = null;
497 if (!scripts.isEmpty()) {
498
499 AwkParser parser = new AwkParser(
500 this.extensionFunctions,
501 settings.isPosix(),
502 isSourceIncludeAllowed());
503 AstNode ast = parser.parse(scripts);
504 lastAst = ast;
505 if (ast != null) {
506
507 ast.semanticAnalysis();
508 ast.semanticAnalysis();
509
510 tuples.setSourceDescription(scripts.get(0).getDescription());
511
512 ast.populateTuples(tuples);
513
514 tuples.postProcess();
515 if (!disableOptimizeParam) {
516 tuples.optimize();
517 }
518
519 parser.populateGlobalVariableNameToOffsetMappings(tuples);
520 }
521 }
522 tuples.freezeMetadata();
523
524 return tuples;
525 }
526
527
528
529
530
531
532 protected boolean isSourceIncludeAllowed() {
533 return true;
534 }
535
536
537
538
539
540
541
542
543 public AwkExpression compileExpression(String expression) throws IOException {
544 return compileExpression(expression, false);
545 }
546
547
548
549
550
551
552
553
554
555 public AwkExpression compileExpression(String expression, boolean disableOptimizeParam) throws IOException {
556 return compileExpression(expression, disableOptimizeParam, new AwkExpression());
557 }
558
559
560
561
562
563
564
565
566
567
568
569 protected final <T extends AwkExpression> T compileExpression(
570 String expression,
571 boolean disableOptimizeParam,
572 T tuples)
573 throws IOException {
574
575 ScriptSource expressionSource = new ScriptSource(
576 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
577 new StringReader(expression));
578
579
580 AwkParser parser = new AwkParser(this.extensionFunctions, settings.isPosix());
581 AstNode ast = parser.parseExpression(expressionSource);
582
583
584
585 if (ast != null) {
586
587 ast.semanticAnalysis();
588
589 ast.semanticAnalysis();
590
591 ast.populateTuples(tuples);
592
593 tuples.postProcess();
594 if (!disableOptimizeParam) {
595 tuples.optimize();
596 }
597
598
599 parser.populateGlobalVariableNameToOffsetMappings(tuples);
600 }
601 tuples.freezeMetadata();
602
603 return tuples;
604 }
605
606
607
608
609
610
611
612
613
614 public Object eval(String expression) throws IOException {
615 return eval(compileExpression(expression));
616 }
617
618
619
620
621
622
623
624
625
626
627 public Object eval(String expression, String input) throws IOException {
628 return eval(compileExpression(expression), input);
629 }
630
631
632
633
634
635
636
637
638
639
640 public Object eval(String expression, InputSource source) throws IOException {
641 return eval(compileExpression(expression), source);
642 }
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662 public AVM prepareEval(String input) throws IOException {
663 String resolvedInput = Objects.requireNonNull(input, "input");
664 AVM evalAvm = createAvm(settings);
665 try {
666 evalAvm.prepareForEval(resolvedInput);
667 return evalAvm;
668 } catch (IOException | RuntimeException e) {
669 try {
670 evalAvm.close();
671 } catch (IOException closeException) {
672 e.addSuppressed(closeException);
673 }
674 throw e;
675 }
676 }
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696 public AVM prepareEval(InputSource source) throws IOException {
697 InputSource resolvedSource = Objects.requireNonNull(source, "source");
698 AVM evalAvm = createAvm(settings);
699 try {
700 if (!evalAvm.prepareForEval(resolvedSource)) {
701 throw new IOException("No record available from source.");
702 }
703 return evalAvm;
704 } catch (IOException | RuntimeException e) {
705 try {
706 evalAvm.close();
707 } catch (IOException closeException) {
708 e.addSuppressed(closeException);
709 }
710 throw e;
711 }
712 }
713
714
715
716
717
718
719
720 protected AVM createAvm(AwkSettings settingsParam) {
721 return createAvm(settingsParam, false);
722 }
723
724
725
726
727
728
729
730
731
732 protected AVM createAvm(AwkSettings settingsParam, boolean profilingEnabled) {
733 return new AVM(settingsParam, this.extensionInstances, profilingEnabled);
734 }
735
736
737
738
739 private static InputStream toInputStream(String input) {
740 if (input == null) {
741 return new ByteArrayInputStream(new byte[0]);
742 }
743 return new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8));
744 }
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768 public final class AwkRunBuilder {
769
770 private AwkProgram compiledProgram;
771 private List<String> scripts;
772 private InputStream inputStream;
773 private InputSource inputSource;
774 private List<String> arguments;
775 private Map<String, Object> variableOverrides;
776 private PrintStream errorStream;
777
778 AwkRunBuilder() {}
779
780 AwkRunBuilder(AwkProgram program) {
781 this.compiledProgram = program;
782 }
783
784
785
786
787
788
789
790
791
792
793 public AwkRunBuilder script(String scriptText) {
794 if (compiledProgram != null) {
795 throw new IllegalStateException("Cannot add scripts when a precompiled program is set");
796 }
797 if (scripts == null) {
798 scripts = new ArrayList<String>();
799 }
800 scripts.add(Objects.requireNonNull(scriptText, "script"));
801 return this;
802 }
803
804
805
806
807
808
809
810 public AwkRunBuilder input(String input) {
811 this.inputStream = toInputStream(input);
812 return this;
813 }
814
815
816
817
818
819
820
821 public AwkRunBuilder input(InputStream input) {
822 this.inputStream = input;
823 return this;
824 }
825
826
827
828
829
830
831
832 public AwkRunBuilder input(InputSource source) {
833 this.inputSource = source;
834 return this;
835 }
836
837
838
839
840
841
842
843 @SuppressFBWarnings("EI_EXPOSE_REP2")
844 public AwkRunBuilder arguments(List<String> args) {
845 this.arguments = args;
846 return this;
847 }
848
849
850
851
852
853
854
855 public AwkRunBuilder arguments(String... args) {
856 this.arguments = Arrays.asList(args);
857 return this;
858 }
859
860
861
862
863
864
865
866 public AwkRunBuilder argument(String arg) {
867 if (this.arguments == null) {
868 this.arguments = new ArrayList<String>();
869 }
870 this.arguments.add(Objects.requireNonNull(arg, "arg"));
871 return this;
872 }
873
874
875
876
877
878
879
880
881
882
883
884
885 public AwkRunBuilder errorStream(PrintStream stream) {
886 this.errorStream = Objects.requireNonNull(stream, "errorStream");
887 return this;
888 }
889
890
891
892
893
894
895
896
897 @SuppressFBWarnings("EI_EXPOSE_REP2")
898 public AwkRunBuilder variables(Map<String, Object> overrides) {
899 this.variableOverrides = overrides;
900 return this;
901 }
902
903
904
905
906
907
908
909
910 public AwkRunBuilder variable(String name, Object value) {
911 if (this.variableOverrides == null) {
912 this.variableOverrides = new LinkedHashMap<String, Object>();
913 }
914 this.variableOverrides
915 .put(
916 Objects.requireNonNull(name, "name"),
917 value);
918 return this;
919 }
920
921
922
923
924
925
926
927
928 public String execute() throws IOException, ExitException {
929 StringBuilder output = new StringBuilder();
930 doExecute(new AppendableAwkSink(output, settings.getLocale()));
931 return output.toString();
932 }
933
934
935
936
937
938
939
940
941 public void execute(AwkSink sink) throws IOException, ExitException {
942 doExecute(Objects.requireNonNull(sink, "sink"));
943 }
944
945
946
947
948
949
950
951
952 public void execute(PrintStream out) throws IOException, ExitException {
953 Objects.requireNonNull(out, "out");
954 doExecute(new OutputStreamAwkSink(out, settings.getLocale()));
955 }
956
957
958
959
960
961
962
963
964 public void execute(OutputStream out) throws IOException, ExitException {
965 doExecute(new OutputStreamAwkSink(toPrintStream(out), settings.getLocale()));
966 }
967
968
969
970
971
972
973
974
975
976 public void execute(Appendable appendable) throws IOException, ExitException {
977 doExecute(
978 new AppendableAwkSink(
979 Objects.requireNonNull(appendable, "appendable"),
980 settings.getLocale()));
981 }
982
983 private void doExecute(AwkSink sink) throws IOException, ExitException {
984 AwkProgram program = resolveProgram();
985 List<String> resolvedArguments = arguments == null ? Collections.<String>emptyList() : arguments;
986 try (AVM avm = createAvm(settings)) {
987 avm.setAwkSink(sink);
988 if (errorStream != null) {
989 avm.setErrorStream(errorStream);
990 avm.setWarningStream(errorStream);
991 } else {
992
993
994
995 avm.setErrorStream(sink.getPrintStream());
996 }
997 try {
998 InputSource resolvedSource;
999 if (inputSource != null) {
1000 resolvedSource = inputSource;
1001 } else {
1002 InputStream in = inputStream != null ? inputStream : new ByteArrayInputStream(new byte[0]);
1003 resolvedSource = new StreamInputSource(in, avm, avm.getJrt());
1004 }
1005 avm.execute(program, resolvedSource, resolvedArguments, variableOverrides);
1006 } catch (ExitException e) {
1007 if (e.getCode() != 0) {
1008 throw e;
1009 }
1010 } finally {
1011 sink.flush();
1012 }
1013 }
1014 }
1015
1016 private AwkProgram resolveProgram() throws IOException {
1017 if (compiledProgram != null) {
1018 return compiledProgram;
1019 }
1020 if (scripts == null || scripts.isEmpty()) {
1021 throw new IllegalStateException("No script or program specified");
1022 }
1023 if (scripts.size() == 1) {
1024 return compile(scripts.get(0));
1025 }
1026 List<ScriptSource> sources = new ArrayList<ScriptSource>(scripts.size());
1027 for (int i = 0; i < scripts.size(); i++) {
1028 sources
1029 .add(
1030 new ScriptSource(
1031 ScriptSource.DESCRIPTION_COMMAND_LINE_SCRIPT,
1032 new StringReader(scripts.get(i))));
1033 }
1034 return compile(sources);
1035 }
1036 }
1037
1038 private static PrintStream toPrintStream(OutputStream out) {
1039 Objects.requireNonNull(out, "outputStream");
1040 if (out instanceof PrintStream) {
1041 return (PrintStream) out;
1042 }
1043 try {
1044 return new PrintStream(out, false, "UTF-8");
1045 } catch (java.io.UnsupportedEncodingException e) {
1046 throw new IllegalStateException(e);
1047 }
1048 }
1049
1050
1051
1052
1053
1054
1055
1056 public static Map<String, JawkExtension> listAvailableExtensions() {
1057 return ExtensionRegistry.listExtensions();
1058 }
1059
1060 private static final class SingleRecordInputSource implements InputSource {
1061
1062 private final String record;
1063
1064 private boolean consumed;
1065
1066 private SingleRecordInputSource(String record) {
1067 this.record = record;
1068 }
1069
1070 @Override
1071 public boolean nextRecord() {
1072 if (consumed || record == null) {
1073 return false;
1074 }
1075 consumed = true;
1076 return true;
1077 }
1078
1079 @Override
1080 public String getRecordText() {
1081 return consumed ? record : null;
1082 }
1083
1084 @Override
1085 public List<String> getFields() {
1086 return null;
1087 }
1088
1089 @Override
1090 public boolean isFromFilenameList() {
1091 return false;
1092 }
1093 }
1094
1095 }