1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24 package com.mindtree.techworks.insight.receiver;
25
26 import java.text.SimpleDateFormat;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.Hashtable;
31 import java.util.Iterator;
32 import java.util.List;
33 import java.util.Map;
34 import java.util.Set;
35 import java.util.TreeMap;
36
37 import org.apache.log4j.Level;
38 import org.apache.log4j.Logger;
39 import org.apache.log4j.helpers.Constants;
40 import org.apache.log4j.rule.ExpressionRule;
41 import org.apache.log4j.rule.Rule;
42 import org.apache.log4j.spi.ThrowableInformation;
43 import org.apache.log4j.spi.LocationInfo;
44 import org.apache.oro.text.perl.Perl5Util;
45 import org.apache.oro.text.regex.MalformedPatternException;
46 import org.apache.oro.text.regex.MatchResult;
47 import org.apache.oro.text.regex.Pattern;
48 import org.apache.oro.text.regex.Perl5Compiler;
49 import org.apache.oro.text.regex.Perl5Matcher;
50
51 import com.mindtree.techworks.insight.model.ReceiverFormat;
52 import com.mindtree.techworks.insight.spi.LogEvent;
53 import com.mindtree.techworks.insight.spi.LogNamespace;
54
55
56
57
58
59
60
61
62
63 public class LogInterpreter {
64
65
66
67
68
69
70
71
72
73
74 private static final String LOGGER = "LOGGER";
75
76
77
78
79 private static final String MESSAGE = "MESSAGE";
80
81
82
83
84 private static final String TIMESTAMP = "TIMESTAMP";
85
86
87
88
89 private static final String NDC = "NDC";
90
91
92
93
94 private static final String LEVEL = "LEVEL";
95
96
97
98
99 private static final String THREAD = "THREAD";
100
101
102
103
104 private static final String CLASS = "CLASS";
105
106
107
108
109 private static final String FILE = "FILE";
110
111
112
113
114 private static final String LINE = "LINE";
115
116
117
118
119 private static final String METHOD = "METHOD";
120
121
122
123
124 private static final String RELATIVETIME = "RELATIVETIME";
125
126
127
128
129 private static final String TIMESTAMP_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS";
130
131
132
133
134 private static final String DEFAULT_LEVEL = "INFO";
135
136
137
138
139 private static final String EXCEPTION_PATTERN = "\tat.*";
140
141
142
143
144 private static final String REGEXP_DEFAULT_WILDCARD = ".*?";
145
146
147
148
149 private static final String REGEXP_GREEDY_WILDCARD = ".+";
150
151
152
153
154 private static final String PATTERN_WILDCARD = "*";
155
156
157
158
159 private static final String DEFAULT_GROUP = "(" + REGEXP_DEFAULT_WILDCARD
160 + ")";
161
162
163
164
165 private static final String GREEDY_GROUP = "(" + REGEXP_GREEDY_WILDCARD
166 + ")";
167
168
169
170
171 private static final String HOSTNAME_PROPERTY_VALUE = "file";
172
173
174
175
176 private static final String VALID_DATEFORMAT_CHAR_PATTERN = "[GyMwWDdFEaHkKhmsSzZ]";
177
178
179
180
181 private static final String PROP_START = "PROP(";
182
183
184
185
186 private static final String PROP_END = ")";
187
188
189
190
191
192
193 private static final int MAX_UNMATCHED_LINE_COUNT = 1000;
194
195
196
197
198 private final Set keywords = new HashSet();
199
200
201
202
203 private List[] matchingKeywords = null;
204
205
206
207
208 private final String newLine = System.getProperty("line.separator");
209
210
211
212
213 private final String[] emptyException = new String[] { "" };
214
215
216
217
218 private SimpleDateFormat[] dateFormat;
219
220
221
222
223 private String filterExpression;
224
225
226
227
228 private final Perl5Util util = new Perl5Util();
229
230
231
232
233 private final Perl5Compiler exceptionCompiler = new Perl5Compiler();
234
235
236
237
238 private final Perl5Matcher exceptionMatcher = new Perl5Matcher();
239
240
241
242
243 private Perl5Compiler compiler = new Perl5Compiler();
244
245
246
247
248 private Pattern[] regexpPattern = null;
249
250
251
252
253 private Perl5Matcher eventMatcher = new Perl5Matcher();
254
255
256
257
258 private Rule expressionRule;
259
260
261
262
263 private final Map currentMap = new HashMap();
264
265
266
267
268 private final List additionalLines = new ArrayList();
269
270
271
272
273 private String regexp;
274
275
276
277
278 private Set greedyKeywords = new HashSet();
279
280
281
282
283 private String timestampPatternText;
284
285
286
287
288 private ReceiverFormat[] receiverFormat = null;
289
290
291
292
293 private String sourceString = null;
294
295
296
297
298 private LogNamespace logNamespace = null;
299
300
301
302
303 private int lastValidReceiverFormatIndex = -1;
304
305
306
307
308
309
310
311
312 public LogInterpreter(LogNamespace logNamespace) {
313
314 this.receiverFormat = logNamespace.getReceiverFormat();
315 this.regexpPattern = new Pattern[receiverFormat.length];
316 this.sourceString = logNamespace.getSourceString();
317 this.logNamespace = logNamespace;
318 this.matchingKeywords = new List[receiverFormat.length];
319 this.dateFormat = new SimpleDateFormat[receiverFormat.length];
320
321 keywords.add(TIMESTAMP);
322 keywords.add(LOGGER);
323 keywords.add(LEVEL);
324 keywords.add(THREAD);
325 keywords.add(CLASS);
326 keywords.add(FILE);
327 keywords.add(LINE);
328 keywords.add(METHOD);
329 keywords.add(MESSAGE);
330 keywords.add(NDC);
331 keywords.add(RELATIVETIME);
332
333 greedyKeywords.add(MESSAGE);
334
335 initializePatternParsing();
336
337 }
338
339
340
341
342
343
344
345
346
347
348 public LogEvent[] parseLogMessage(String line) {
349
350 if (line == null) {
351
352 LogEvent event = buildEvent(lastValidReceiverFormatIndex);
353 if (event != null) {
354 if (passesExpression(event)) {
355 LogEvent[] lastEvent = new LogEvent[1];
356 lastEvent[0] = event;
357 return lastEvent;
358 }
359 }
360 return new LogEvent[0];
361 }
362
363 ArrayList eventsList = new ArrayList();
364
365
366 int validReceiverFormat = -1;
367 boolean isValidRegexpPattern = false;
368 for (int i = 0; i < regexpPattern.length; i++) {
369 if (eventMatcher.matches(line, regexpPattern[i])) {
370 validReceiverFormat = i;
371 this.lastValidReceiverFormatIndex = validReceiverFormat;
372 LogEvent event = buildEvent(validReceiverFormat);
373 if (event != null) {
374 if (passesExpression(event)) {
375 eventsList.add(event);
376 }
377 }
378 currentMap.putAll(processEvent(eventMatcher.getMatch(),
379 validReceiverFormat));
380 isValidRegexpPattern = true;
381 break;
382 }
383 validReceiverFormat = -1;
384 }
385 if (!isValidRegexpPattern) {
386
387 additionalLines.add(line);
388
389
390
391 if (additionalLines.size() > MAX_UNMATCHED_LINE_COUNT) {
392 additionalLines.clear();
393 }
394 }
395 return (LogEvent[]) eventsList.toArray(new LogEvent[0]);
396 }
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411 private int getExceptionLine() {
412
413 try {
414 Pattern exceptionPattern = exceptionCompiler
415 .compile(EXCEPTION_PATTERN);
416 for (int i = 0; i < additionalLines.size(); i++) {
417 if (exceptionMatcher.matches((String) additionalLines.get(i),
418 exceptionPattern)) {
419 return i - 1;
420 }
421 }
422 } catch (MalformedPatternException mpe) {
423 mpe.printStackTrace();
424 }
425 return -1;
426 }
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442 private String buildMessage(String firstMessageLine, int exceptionLine) {
443
444 if (additionalLines.size() == 0 || exceptionLine == 0) {
445 return firstMessageLine;
446 }
447 StringBuffer message = new StringBuffer(firstMessageLine);
448 int linesToProcess = (exceptionLine == -1 ? additionalLines.size()
449 : exceptionLine);
450
451 for (int i = 0; i < linesToProcess; i++) {
452 message.append(newLine);
453 message.append(additionalLines.get(i));
454 }
455 return message.toString();
456 }
457
458
459
460
461
462
463
464
465
466
467
468 private String[] buildException(int exceptionLine) {
469
470 if (exceptionLine == -1) {
471 return emptyException;
472 }
473 String[] exception = new String[additionalLines.size() - exceptionLine];
474 for (int i = 0; i < additionalLines.size() - exceptionLine; i++) {
475 exception[i] = (String) additionalLines.get(i + exceptionLine);
476 }
477 return exception;
478 }
479
480
481
482
483
484
485
486
487
488
489
490 private LogEvent buildEvent(int validReceiverFormat) {
491
492 LogEvent event = null;
493
494 if (currentMap.size() == 0) {
495 return event;
496 }
497 int exceptionLine = getExceptionLine();
498 String[] exception = buildException(exceptionLine);
499
500
501 if (additionalLines.size() > 0 && exceptionLine != 0) {
502 currentMap.put(MESSAGE, buildMessage((String) currentMap
503 .get(MESSAGE), exceptionLine));
504 }
505
506 if (currentMap.size() > 0) {
507 event = convertToEvent(currentMap, exception, validReceiverFormat);
508 }
509 this.currentMap.clear();
510 this.additionalLines.clear();
511
512 return event;
513 }
514
515
516
517
518
519
520
521
522 private boolean passesExpression(LogEvent event) {
523
524 if (event != null) {
525 if (expressionRule != null) {
526 return (expressionRule.evaluate(event));
527 }
528 }
529 return true;
530 }
531
532
533
534
535
536
537
538
539
540
541
542
543 private Map processEvent(MatchResult result, int validReceiverFormat) {
544
545 Map map = new HashMap();
546
547 for (int i = 1; i < result.groups(); i++) {
548 map.put(matchingKeywords[validReceiverFormat].get(i - 1), result
549 .group(i));
550 }
551 return map;
552 }
553
554
555
556
557
558
559 private String convertTimestamp(ReceiverFormat receiverFormat) {
560
561 String timestampFormat = receiverFormat.getTimeStampPattern() == null ? TIMESTAMP_FORMAT
562 : receiverFormat.getTimeStampPattern();
563 return util.substitute("s/" + VALID_DATEFORMAT_CHAR_PATTERN
564 + "/\\\\w/g", timestampFormat);
565 }
566
567
568
569
570 private void initializePatternParsing() {
571 ReceiverFormat[] receiverFormat = this.getReceiverFormat();
572 for (int i = 0; i < receiverFormat.length; i++) {
573 String timestampFormat = receiverFormat[i].getTimeStampPattern() == null ? TIMESTAMP_FORMAT
574 : receiverFormat[i].getTimeStampPattern();
575 if (timestampFormat != null) {
576 dateFormat[i] = new SimpleDateFormat(timestampFormat);
577 timestampPatternText = convertTimestamp(receiverFormat[i]);
578 }
579
580 try {
581 if (filterExpression != null) {
582 expressionRule = ExpressionRule.getRule(filterExpression);
583 }
584 } catch (Exception e) {
585 e.printStackTrace();
586 }
587
588 Map keywordMap = new TreeMap();
589
590 String newPattern = receiverFormat[i].getLogPattern();
591
592
593
594
595
596
597
598 int index = 0;
599 String current = newPattern;
600 while (index > -1) {
601 index = current.indexOf(PROP_START);
602 if (index > -1) {
603 String currentProp = current.substring(current
604 .indexOf(PROP_START));
605 String prop = currentProp.substring(0, currentProp
606 .indexOf(PROP_END) + 1);
607 current = current
608 .substring(current.indexOf(currentProp) + 1);
609 String shortProp = prop.substring(PROP_START.length(), prop
610 .length() - 1);
611 keywordMap.put(new Integer(index), shortProp);
612 newPattern = replace(prop, shortProp, newPattern);
613 }
614 }
615
616 newPattern = replaceMetaChars(newPattern);
617 newPattern = replace(PATTERN_WILDCARD, REGEXP_DEFAULT_WILDCARD,
618 newPattern);
619
620
621
622
623
624
625
626
627 Iterator iter = keywords.iterator();
628 while (iter.hasNext()) {
629 String keyword = (String) iter.next();
630 int index2 = newPattern.indexOf(keyword);
631 if (index2 > -1) {
632 keywordMap.put(new Integer(index2), keyword);
633 }
634 }
635
636
637 matchingKeywords[i] = new ArrayList();
638 matchingKeywords[i].addAll(keywordMap.values());
639
640
641
642
643
644 String currentPattern = newPattern;
645 Iterator iter2 = matchingKeywords[i].iterator();
646 while (iter2.hasNext()) {
647 String keyword = (String) iter2.next();
648 if (TIMESTAMP.equals(keyword)) {
649 currentPattern = replace(keyword, "("
650 + timestampPatternText + ")", currentPattern);
651 } else {
652 currentPattern = replace(keyword, greedyKeywords
653 .contains(keyword) ? GREEDY_GROUP : DEFAULT_GROUP,
654 currentPattern);
655 }
656 }
657 this.regexp = currentPattern;
658 try {
659 this.regexpPattern[i] = compiler.compile(regexp);
660 } catch (MalformedPatternException mpe) {
661 throw new RuntimeException("Bad pattern: " + regexp);
662 }
663 }
664 }
665
666
667
668
669
670
671
672
673
674 private String replace(String pattern, String replacement, String input) {
675
676 return util.substitute("s/" + Perl5Compiler.quotemeta(pattern) + "/"
677 + Perl5Compiler.quotemeta(replacement) + "/g", input);
678 }
679
680
681
682
683
684
685
686
687 private String replaceMetaChars(String input) {
688
689 input = replace("(", "\\(", input);
690 input = replace(")", "\\)", input);
691 input = replace("[", "\\[", input);
692 input = replace("]", "\\]", input);
693 input = replace("{", "\\{", input);
694 input = replace("}", "\\}", input);
695 input = replace("#", "\\#", input);
696 return input;
697 }
698
699
700
701
702
703
704
705
706
707
708 private LogEvent convertToEvent(Map fieldMap, String[] exception,
709 int validReceiverFormat) {
710
711 if (fieldMap == null) {
712 return null;
713 }
714
715
716 if (!fieldMap.containsKey(LOGGER)) {
717 fieldMap.put(LOGGER, "Unknown");
718 }
719 if (exception == null) {
720 exception = emptyException;
721 }
722
723 Logger logger = null;
724 long timeStamp = 0L;
725 long relativeTime = 0L;
726 String level = null;
727 String threadName = null;
728 Object message = null;
729 String ndc = null;
730 String className = null;
731 String methodName = null;
732 String eventFileName = null;
733 String lineNumber = null;
734 Hashtable properties = new Hashtable();
735
736 logger = Logger.getLogger((String) fieldMap.remove(LOGGER));
737
738 if ((dateFormat != null) && fieldMap.containsKey(TIMESTAMP)) {
739 try {
740
741 timeStamp = dateFormat[validReceiverFormat].parse(
742 ((String) fieldMap.remove(TIMESTAMP)).trim()).getTime();
743 } catch (Exception e) {
744 e.printStackTrace();
745 }
746 }
747
748 if (fieldMap.containsKey(RELATIVETIME)) {
749
750 relativeTime = Long.parseLong(((String) fieldMap
751 .remove(RELATIVETIME)).trim());
752 }
753
754 if (fieldMap.containsKey(LEVEL)) {
755 level = (String) fieldMap.remove(LEVEL);
756
757 } else {
758 level = DEFAULT_LEVEL;
759 }
760 Level levelImpl = Level.toLevel(level.trim());
761
762 threadName = (String) fieldMap.remove(THREAD);
763 message = (String) fieldMap.remove(MESSAGE);
764 ndc = (String) fieldMap.remove(NDC);
765 className = (String) fieldMap.remove(CLASS);
766 methodName = (String) fieldMap.remove(METHOD);
767 eventFileName = (String) fieldMap.remove(FILE);
768 lineNumber = (String) fieldMap.remove(LINE);
769
770 properties.put(Constants.HOSTNAME_KEY, HOSTNAME_PROPERTY_VALUE);
771 properties.put(Constants.APPLICATION_KEY, this.getSourceString());
772
773
774 properties.putAll(fieldMap);
775
776 LocationInfo info = null;
777
778 if ((eventFileName != null) || (className != null)
779 || (methodName != null) || (lineNumber != null)) {
780 info = new LocationInfo(eventFileName, className, methodName,
781 lineNumber);
782 } else {
783 info = LocationInfo.NA_LOCATION_INFO;
784 }
785
786
787
788
789
790
791 LogEvent event = new LogEvent();
792 event.setLogger(logger);
793 event.setTimeStamp(timeStamp);
794 event.setRelativeTime(relativeTime);
795 event.setLevel(levelImpl);
796 event.setMessage(message);
797 event.setThreadName(threadName);
798 event.setThrowableInformation(new ThrowableInformation(exception));
799 event.setLocationInformation(info);
800 event.setNDC(ndc);
801 event.setProperties(properties);
802 event.setNamespace(logNamespace);
803 return event;
804 }
805
806
807
808
809
810
811
812
813 public void setLogNamespace(LogNamespace logNamespace) {
814
815 this.logNamespace = logNamespace;
816 }
817
818
819
820
821 public ReceiverFormat[] getReceiverFormat() {
822
823 return receiverFormat;
824 }
825
826
827
828
829 public String getSourceString() {
830
831 return sourceString;
832 }
833
834 }