1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 package org.apache.hadoop.chukwa.inputtools.log4j;
26
27
28 import java.io.File;
29 import java.io.FilenameFilter;
30 import java.io.IOException;
31 import java.text.SimpleDateFormat;
32 import java.util.ArrayList;
33 import java.util.Calendar;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.GregorianCalendar;
37 import java.util.List;
38 import java.util.Locale;
39 import java.util.TimeZone;
40 import java.util.regex.Pattern;
41
42 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
43 import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
44 import org.apache.hadoop.chukwa.util.AdaptorNamingUtils;
45 import org.apache.hadoop.chukwa.util.RecordConstants;
46 import org.apache.log4j.FileAppender;
47 import org.apache.log4j.Layout;
48 import org.apache.log4j.Logger;
49 import org.apache.log4j.helpers.LogLog;
50 import org.apache.log4j.spi.LoggingEvent;
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158 public class ChukwaDailyRollingFileAppender extends FileAppender {
159
160 static Logger log = Logger.getLogger(ChukwaDailyRollingFileAppender.class);
161
162
163 static final int TOP_OF_TROUBLE = -1;
164 static final int TOP_OF_MINUTE = 0;
165 static final int TOP_OF_HOUR = 1;
166 static final int HALF_DAY = 2;
167 static final int TOP_OF_DAY = 3;
168 static final int TOP_OF_WEEK = 4;
169 static final int TOP_OF_MONTH = 5;
170
171 static final String adaptorType = ChukwaAgentController.CharFileTailUTF8NewLineEscaped;
172
173 static final Object lock = new Object();
174 static String lastRotation = "";
175
176
177
178
179
180 private String datePattern = "'.'yyyy-MM-dd";
181
182
183
184
185
186
187
188
189
190 private String scheduledFilename;
191
192
193
194
195 private long nextCheck = System.currentTimeMillis() - 1;
196
197
198
199
200 private String cleanUpRegex = null;
201
202
203
204
205 private int maxBackupIndex = 10;
206
207 private ClientFinalizer clientFinalizer = null;
208
209 boolean hasBeenActivated = false;
210 Date now = new Date();
211
212 SimpleDateFormat sdf;
213
214 RollingCalendar rc = new RollingCalendar();
215
216 int checkPeriod = TOP_OF_TROUBLE;
217
218 ChukwaAgentController chukwaClient;
219 boolean chukwaClientIsNull = true;
220 static final Object chukwaLock = new Object();
221
222 String chukwaClientHostname;
223 int chukwaClientPortNum;
224 long chukwaClientConnectNumRetry;
225 long chukwaClientConnectRetryInterval;
226
227 String recordType;
228
229
230 static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT");
231
232
233
234
235
236 public ChukwaDailyRollingFileAppender() throws IOException {
237 super();
238 }
239
240
241
242
243
244
245
246
247
248
249
250 public ChukwaDailyRollingFileAppender(Layout layout, String filename,
251 String datePattern) throws IOException {
252 super(layout, filename, true);
253 System.out
254 .println("Daily Rolling File Appender successfully registered file with agent: "
255 + filename);
256 this.datePattern = datePattern;
257 }
258
259
260
261
262
263
264 public void setDatePattern(String pattern) {
265 datePattern = pattern;
266 }
267
268
269
270 public String getDatePattern() {
271 return datePattern;
272 }
273
274 public String getRecordType() {
275 if (recordType != null)
276 return recordType;
277 else
278 return "unknown";
279 }
280
281 public void setRecordType(String recordType) {
282 this.recordType = recordType;
283 }
284
285 public void activateOptions() {
286
287
288 if (!hasBeenActivated)
289 { return;}
290
291 super.activateOptions();
292 if (datePattern != null && fileName != null) {
293 now.setTime(System.currentTimeMillis());
294 sdf = new SimpleDateFormat(datePattern);
295 int type = computeCheckPeriod();
296 printPeriodicity(type);
297 rc.setType(type);
298 File file = new File(fileName);
299 scheduledFilename = fileName + sdf.format(new Date(file.lastModified()));
300
301 } else {
302 LogLog
303 .error("Either File or DatePattern options are not set for appender ["
304 + name + "].");
305 }
306 }
307
308 void printPeriodicity(int type) {
309 switch (type) {
310 case TOP_OF_MINUTE:
311 LogLog.debug("Appender [" + name + "] to be rolled every minute.");
312 break;
313 case TOP_OF_HOUR:
314 LogLog
315 .debug("Appender [" + name + "] to be rolled on top of every hour.");
316 break;
317 case HALF_DAY:
318 LogLog.debug("Appender [" + name
319 + "] to be rolled at midday and midnight.");
320 break;
321 case TOP_OF_DAY:
322 LogLog.debug("Appender [" + name + "] to be rolled at midnight.");
323 break;
324 case TOP_OF_WEEK:
325 LogLog.debug("Appender [" + name + "] to be rolled at start of week.");
326 break;
327 case TOP_OF_MONTH:
328 LogLog.debug("Appender [" + name
329 + "] to be rolled at start of every month.");
330 break;
331 default:
332 LogLog.warn("Unknown periodicity for appender [" + name + "].");
333 }
334 }
335
336
337
338
339
340
341
342
343
344
345 int computeCheckPeriod() {
346 RollingCalendar rollingCalendar = new RollingCalendar(gmtTimeZone,
347 Locale.ENGLISH);
348
349 Date epoch = new Date(0);
350 if (datePattern != null) {
351 for (int i = TOP_OF_MINUTE; i <= TOP_OF_MONTH; i++) {
352 SimpleDateFormat simpleDateFormat = new SimpleDateFormat(datePattern);
353 simpleDateFormat.setTimeZone(gmtTimeZone);
354
355 String r0 = simpleDateFormat.format(epoch);
356 rollingCalendar.setType(i);
357 Date next = new Date(rollingCalendar.getNextCheckMillis(epoch));
358 String r1 = simpleDateFormat.format(next);
359
360 if (r0 != null && r1 != null && !r0.equals(r1)) {
361 return i;
362 }
363 }
364 }
365 return TOP_OF_TROUBLE;
366 }
367
368
369
370
371 void rollOver() throws IOException {
372
373
374 if (datePattern == null) {
375 errorHandler.error("Missing DatePattern option in rollOver().");
376 return;
377 }
378
379 String datedFilename = fileName + sdf.format(now);
380
381
382
383 if (scheduledFilename.equals(datedFilename)) {
384 return;
385 }
386
387
388 this.closeFile();
389
390 File target = new File(scheduledFilename);
391 if (target.exists()) {
392 if(!target.delete()) {
393 LogLog.warn("Unable to remove: "+target.getAbsolutePath());
394 };
395 }
396
397 File file = new File(fileName);
398
399 boolean result = file.renameTo(target);
400 if (result) {
401 LogLog.debug(fileName + " -> " + scheduledFilename);
402 } else {
403 LogLog.error("Failed to rename [" + fileName + "] to ["
404 + scheduledFilename + "].");
405 }
406
407 try {
408
409
410 this.setFile(fileName, false, this.bufferedIO, this.bufferSize);
411 } catch (IOException e) {
412 errorHandler.error("setFile(" + fileName + ", false) call failed.");
413 }
414 scheduledFilename = datedFilename;
415 cleanUp();
416 }
417
418 public synchronized String getCleanUpRegex() {
419 return cleanUpRegex;
420 }
421
422 protected synchronized void setCleanUpRegex(String cleanUpRegex) {
423 this.cleanUpRegex = cleanUpRegex;
424 }
425
426 public int getMaxBackupIndex() {
427 return maxBackupIndex;
428 }
429
430 public void setMaxBackupIndex(int maxBackupIndex) {
431 this.maxBackupIndex = maxBackupIndex;
432 }
433
434 protected synchronized void cleanUp() {
435 String regex = "";
436 try {
437 File actualFile = new File(fileName);
438
439 String directoryName = actualFile.getParent();
440 String actualFileName = actualFile.getName();
441 File dirList = new File(directoryName);
442
443 if (cleanUpRegex == null || !cleanUpRegex.contains("$fileName")) {
444 LogLog
445 .error("cleanUpRegex == null || !cleanUpRegex.contains(\"$fileName\")");
446 return;
447 }
448 regex = cleanUpRegex.replace("$fileName", actualFileName);
449 String[] dirFiles = dirList.list(new LogFilter(actualFileName, regex));
450
451 List<String> files = new ArrayList<String>();
452 if(dirFiles!=null) {
453 for (String file : dirFiles) {
454 files.add(file);
455 }
456 }
457 Collections.sort(files);
458
459 while (files.size() > maxBackupIndex) {
460 String file = files.remove(0);
461 File f = new File(directoryName + "/" + file);
462 if(!f.delete()) {
463 LogLog.warn("Cannot remove: " + file);
464 }
465 }
466 } catch (Exception e) {
467 errorHandler
468 .error("cleanUp(" + fileName + "," + regex + ") call failed.");
469 }
470 }
471
472 private static class LogFilter implements FilenameFilter {
473 private Pattern p = null;
474 private String logFile = null;
475
476 public LogFilter(String logFile, String regex) {
477 this.logFile = logFile;
478 p = Pattern.compile(regex);
479 }
480
481 @Override
482 public boolean accept(File dir, String name) {
483
484 if (name.intern() == this.logFile.intern()) {
485 return false;
486 }
487
488 if (!name.startsWith(logFile)) {
489 return false;
490 }
491 return p.matcher(name).find();
492 }
493 }
494
495
496
497
498 @Override
499 protected boolean checkEntryConditions() {
500 synchronized(chukwaLock) {
501 if (!hasBeenActivated) {
502 hasBeenActivated = true;
503 activateOptions();
504 }
505 }
506 return super.checkEntryConditions();
507 }
508
509
510
511
512
513
514
515
516 protected void subAppend(LoggingEvent event) {
517 try {
518
519
520
521
522
523
524
525 if (chukwaClientIsNull) {
526 synchronized (chukwaLock) {
527
528 String log4jFileName = getFile();
529 String recordType = getRecordType();
530
531 long currentLength = 0L;
532 try {
533 File fooLog = new File(log4jFileName);
534 log4jFileName = fooLog.getAbsolutePath();
535 currentLength = fooLog.length();
536 } catch (Throwable e) {
537 log.warn("Exception while trying to get current file size for " + log4jFileName);
538 currentLength = 0L;
539 }
540
541 if (chukwaClient == null) {
542 if (getChukwaClientHostname() != null
543 && getChukwaClientPortNum() != 0) {
544 chukwaClient = new ChukwaAgentController(
545 getChukwaClientHostname(), getChukwaClientPortNum());
546 log.debug("setup adaptor with hostname "
547 + getChukwaClientHostname() + " and portnum "
548 + getChukwaClientPortNum());
549 } else {
550 chukwaClient = new ChukwaAgentController();
551 log
552 .debug("setup adaptor with no args, which means it used its defaults");
553 }
554
555 chukwaClientIsNull = false;
556
557
558
559
560
561
562
563
564 long retryInterval = chukwaClientConnectRetryInterval;
565 if (retryInterval == 0) {
566 retryInterval = 1000 * 60 * 30;
567 }
568 long numRetries = chukwaClientConnectNumRetry;
569 if (numRetries == 0) {
570 numRetries = 48;
571 }
572
573 String name = AdaptorNamingUtils.synthesizeAdaptorID
574 (ChukwaAgentController.CharFileTailUTF8NewLineEscaped, recordType, log4jFileName);
575
576 String adaptorID = chukwaClient.addByName(name, ChukwaAgentController.CharFileTailUTF8NewLineEscaped,
577 recordType,currentLength + " " + log4jFileName, currentLength,
578 numRetries, retryInterval);
579
580
581 clientFinalizer = new ClientFinalizer(chukwaClient);
582 Runtime.getRuntime().addShutdownHook(clientFinalizer);
583
584 if (adaptorID != null) {
585 log.debug("Added file tailing adaptor to chukwa agent for file "
586 + log4jFileName + ", adaptorId:" + adaptorID
587 + " using this recordType :" + recordType
588 + ", starting at offset:" + currentLength);
589 } else {
590 log.debug("Chukwa adaptor not added, addFile(" + log4jFileName
591 + ") returned, current offset: " + currentLength);
592 }
593
594 }
595 }
596 }
597
598 long n = System.currentTimeMillis();
599 if (n >= nextCheck) {
600 now.setTime(n);
601 nextCheck = rc.getNextCheckMillis(now);
602 try {
603 rollOver();
604 } catch (IOException ioe) {
605 LogLog.error("rollOver() failed.", ioe);
606 }
607 }
608
609 boolean written = false;
610 if(layout.ignoresThrowable()) {
611 String[] s = event.getThrowableStrRep();
612 if (s != null) {
613 int len = s.length;
614 StringBuilder sb = new StringBuilder();
615 sb.append(this.layout.format(event));
616 for(int i = 0; i < len; i++) {
617 sb.append(s[i]).append("\n");
618 }
619
620 written = true;
621 this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",sb.toString()));
622 }
623 }
624
625 if (!written) {
626
627 this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",this.layout.format(event)));
628 }
629
630 if (this.immediateFlush) {
631 this.qw.flush();
632 }
633 } catch (Throwable e) {
634 System.err.println("Exception in ChukwaRollingAppender: "
635 + e.getMessage());
636 e.printStackTrace();
637 }
638
639 }
640
641 public String getChukwaClientHostname() {
642 return chukwaClientHostname;
643 }
644
645 public void setChukwaClientHostname(String chukwaClientHostname) {
646 this.chukwaClientHostname = chukwaClientHostname;
647 }
648
649 public int getChukwaClientPortNum() {
650 return chukwaClientPortNum;
651 }
652
653 public void setChukwaClientPortNum(int chukwaClientPortNum) {
654 this.chukwaClientPortNum = chukwaClientPortNum;
655 }
656
657 public void setChukwaClientConnectNumRetry(int i) {
658 this.chukwaClientConnectNumRetry = i;
659 }
660
661 public void setChukwaClientConnectRetryInterval(long i) {
662 this.chukwaClientConnectRetryInterval = i;
663 }
664
665 }
666
667
668
669
670
671
672
673 class RollingCalendar extends GregorianCalendar {
674
675
676
677
678 private static final long serialVersionUID = 2153481574198792767L;
679 int type = ChukwaDailyRollingFileAppender.TOP_OF_TROUBLE;
680
681 RollingCalendar() {
682 super();
683 }
684
685 RollingCalendar(TimeZone tz, Locale locale) {
686 super(tz, locale);
687 }
688
689 void setType(int type) {
690 this.type = type;
691 }
692
693 public long getNextCheckMillis(Date now) {
694 return getNextCheckDate(now).getTime();
695 }
696
697 public Date getNextCheckDate(Date now) {
698 this.setTime(now);
699
700 switch (type) {
701 case ChukwaDailyRollingFileAppender.TOP_OF_MINUTE:
702 this.set(Calendar.SECOND, 0);
703 this.set(Calendar.MILLISECOND, 0);
704 this.add(Calendar.MINUTE, 1);
705 break;
706 case ChukwaDailyRollingFileAppender.TOP_OF_HOUR:
707 this.set(Calendar.MINUTE, 0);
708 this.set(Calendar.SECOND, 0);
709 this.set(Calendar.MILLISECOND, 0);
710 this.add(Calendar.HOUR_OF_DAY, 1);
711 break;
712 case ChukwaDailyRollingFileAppender.HALF_DAY:
713 this.set(Calendar.MINUTE, 0);
714 this.set(Calendar.SECOND, 0);
715 this.set(Calendar.MILLISECOND, 0);
716 int hour = get(Calendar.HOUR_OF_DAY);
717 if (hour < 12) {
718 this.set(Calendar.HOUR_OF_DAY, 12);
719 } else {
720 this.set(Calendar.HOUR_OF_DAY, 0);
721 this.add(Calendar.DAY_OF_MONTH, 1);
722 }
723 break;
724 case ChukwaDailyRollingFileAppender.TOP_OF_DAY:
725 this.set(Calendar.HOUR_OF_DAY, 0);
726 this.set(Calendar.MINUTE, 0);
727 this.set(Calendar.SECOND, 0);
728 this.set(Calendar.MILLISECOND, 0);
729 this.add(Calendar.DATE, 1);
730 break;
731 case ChukwaDailyRollingFileAppender.TOP_OF_WEEK:
732 this.set(Calendar.DAY_OF_WEEK, getFirstDayOfWeek());
733 this.set(Calendar.HOUR_OF_DAY, 0);
734 this.set(Calendar.SECOND, 0);
735 this.set(Calendar.MILLISECOND, 0);
736 this.add(Calendar.WEEK_OF_YEAR, 1);
737 break;
738 case ChukwaDailyRollingFileAppender.TOP_OF_MONTH:
739 this.set(Calendar.DATE, 1);
740 this.set(Calendar.HOUR_OF_DAY, 0);
741 this.set(Calendar.SECOND, 0);
742 this.set(Calendar.MILLISECOND, 0);
743 this.add(Calendar.MONTH, 1);
744 break;
745 default:
746 throw new IllegalStateException("Unknown periodicity type.");
747 }
748 return getTime();
749 }
750
751 @Override
752 public boolean equals(Object o) {
753 return super.equals(o);
754 }
755
756 @Override
757 public int hashCode() {
758 return super.hashCode();
759 }
760 }