'use strict';
/*!
* FileStreamRotator
* Copyright(c) 2012-2017 Holiday Extras.
* Copyright(c) 2017 Roger C.
* MIT Licensed
*/
/**
* Module dependencies.
*/
var fs = require('fs');
var path = require('path');
var moment = require('moment');
var crypto = require('crypto');
var EventEmitter = require('events');
/**
* FileStreamRotator:
*
* Returns a file stream that auto-rotates based on date.
*
* Options:
*
* - `filename` Filename including full path used by the stream
*
* - `frequency` How often to rotate. Options are 'daily', 'custom' and 'test'. 'test' rotates every minute.
* If frequency is set to none of the above, a YYYYMMDD string will be added to the end of the filename.
*
* - `verbose` If set, it will log to STDOUT when it rotates files and name of log file. Default is TRUE.
*
* - `date_format` Format as used in moment.js http://momentjs.com/docs/#/displaying/format/. The result is used to replace
* the '%DATE%' placeholder in the filename.
* If using 'custom' frequency, it is used to trigger file change when the string representation changes.
*
* - `size` Max size of the file after which it will rotate. It can be combined with frequency or date format.
* The size units are 'k', 'm' and 'g'. Units need to directly follow a number e.g. 1g, 100m, 20k.
*
* - `max_logs` Max number of logs to keep. If not set, it won't remove past logs. It uses its own log audit file
* to keep track of the log files in a json format. It won't delete any file not contained in it.
* It can be a number of files or number of days. If using days, add 'd' as the suffix.
*
* - `audit_file` Location to store the log audit file. If not set, it will be stored in the root of the application.
*
* - `end_stream` End stream (true) instead of the default behaviour of destroy (false). Set value to true if when writing to the
* stream in a loop, if the application terminates or log rotates, data pending to be flushed might be lost.
*
* - `file_options` An object passed to the stream. This can be used to specify flags, encoding, and mode.
* See https://nodejs.org/api/fs.html#fs_fs_createwritestream_path_options. Default `{ flags: 'a' }`.
*
* - `utc` Use UTC time for date in filename. Defaults to 'FALSE'
*
* - `extension` File extension to be appended to the filename. This is useful when using size restrictions as the rotation
* adds a count (1,2,3,4,...) at the end of the filename when the required size is met.
*
* - `watch_log` Watch the current file being written to and recreate it in case of accidental deletion. Defaults to 'FALSE'
*
* - `create_symlink` Create a tailable symlink to the current active log file. Defaults to 'FALSE'
*
* - `symlink_name` Name to use when creating the symbolic link. Defaults to 'current.log'
*
* - `audit_hash_type` Use specified hashing algorithm for audit. Defaults to 'md5'. Use 'sha256' for FIPS compliance.
*
* To use with Express / Connect, use as below.
*
* var rotatingLogStream = require('FileStreamRotator').getStream({filename:"/tmp/test.log", frequency:"daily", verbose: false})
* app.use(express.logger({stream: rotatingLogStream, format: "default"}));
*
* @param {Object} options
* @return {Object}
* @api public
*/
var FileStreamRotator = {};
module.exports = FileStreamRotator;
var staticFrequency = ['daily', 'test', 'm', 'h', 'custom'];
var DATE_FORMAT = ('YYYYMMDDHHmm');
/**
* Returns frequency metadata for minute/hour rotation
* @param type
* @param num
* @returns {*}
* @private
*/
var _checkNumAndType = function (type, num) {
if (typeof num == 'number') {
switch (type) {
case 'm':
if (num < 0 || num > 60) {
return false;
}
break;
case 'h':
if (num < 0 || num > 24) {
return false;
}
break;
}
return {type: type, digit: num};
}
}
/**
* Returns frequency metadata for defined frequency
* @param freqType
* @returns {*}
* @private
*/
var _checkDailyAndTest = function (freqType) {
switch (freqType) {
case 'custom':
case 'daily':
return {type: freqType, digit: undefined};
break;
case 'test':
return {type: freqType, digit: 0};
}
return false;
}
/**
* Returns frequency metadata
* @param frequency
* @returns {*}
*/
FileStreamRotator.getFrequency = function (frequency) {
var _f = frequency.toLowerCase().match(/^(\d+)([mh])$/)
if(_f){
return _checkNumAndType(_f[2], parseInt(_f[1]));
}
var dailyOrTest = _checkDailyAndTest(frequency);
if (dailyOrTest) {
return dailyOrTest;
}
return false;
}
/**
* Returns a number based on the option string
* @param size
* @returns {Number}
*/
FileStreamRotator.parseFileSize = function (size) {
if(size && typeof size == "string"){
var _s = size.toLowerCase().match(/^((?:0\.)?\d+)([kmg])$/);
if(_s){
switch(_s[2]){
case 'k':
return _s[1]*1024
case 'm':
return _s[1]*1024*1024
case 'g':
return _s[1]*1024*1024*1024
}
}
}
return null;
};
/**
* Returns date string for a given format / date_format
* @param format
* @param date_format
* @param {boolean} utc
* @returns {string}
*/
FileStreamRotator.getDate = function (format, date_format, utc) {
date_format = date_format || DATE_FORMAT;
let currentMoment = utc ? moment.utc() : moment().local()
if (format && staticFrequency.indexOf(format.type) !== -1) {
switch (format.type) {
case 'm':
var minute = Math.floor(currentMoment.minutes() / format.digit) * format.digit;
return currentMoment.minutes(minute).format(date_format);
break;
case 'h':
var hour = Math.floor(currentMoment.hour() / format.digit) * format.digit;
return currentMoment.hour(hour).format(date_format);
break;
case 'daily':
case 'custom':
case 'test':
return currentMoment.format(date_format);
}
}
return currentMoment.format(date_format);
}
/**
* Read audit json object from disk or return new object or null
* @param max_logs
* @param audit_file
* @param log_file
* @returns {Object} auditLogSettings
* @property {Object} auditLogSettings.keep
* @property {Boolean} auditLogSettings.keep.days
* @property {Number} auditLogSettings.keep.amount
* @property {String} auditLogSettings.auditLog
* @property {Array} auditLogSettings.files
* @property {String} auditLogSettings.hashType
*/
FileStreamRotator.setAuditLog = function (max_logs, audit_file, log_file){
var _rtn = null;
if(max_logs){
var use_days = max_logs.toString().substr(-1);
var _num = max_logs.toString().match(/^(\d+)/);
if(Number(_num[1]) > 0) {
var baseLog = path.dirname(log_file.replace(/%DATE%.+/,"_filename"));
try{
if(audit_file){
var full_path = path.resolve(audit_file);
_rtn = JSON.parse(fs.readFileSync(full_path, { encoding: 'utf-8' }));
}else{
var full_path = path.resolve(baseLog + "/" + ".audit.json")
_rtn = JSON.parse(fs.readFileSync(full_path, { encoding: 'utf-8' }));
}
}catch(e){
if(e.code !== "ENOENT"){
return null;
}
_rtn = {
keep: {
days: false,
amount: Number(_num[1])
},
auditLog: audit_file || baseLog + "/" + ".audit.json",
files: []
};
}
_rtn.keep = {
days: use_days === 'd',
amount: Number(_num[1])
};
}
}
return _rtn;
};
/**
* Write audit json object to disk
* @param {Object} audit
* @param {Object} audit.keep
* @param {Boolean} audit.keep.days
* @param {Number} audit.keep.amount
* @param {String} audit.auditLog
* @param {Array} audit.files
* @param {String} audit.hashType
* @param {Boolean} verbose
*/
FileStreamRotator.writeAuditLog = function(audit, verbose){
try{
mkDirForFile(audit.auditLog);
fs.writeFileSync(audit.auditLog, JSON.stringify(audit,null,4));
}catch(e){
if (verbose) {
console.error(new Date(),"[FileStreamRotator] Failed to store log audit at:", audit.auditLog,"Error:", e);
}
}
};
/**
* Removes old log file
* @param file
* @param file.hash
* @param file.name
* @param file.date
* @param file.hashType
* @param {Boolean} verbose
*/
function removeFile(file, verbose){
if(file.hash === crypto.createHash(file.hashType).update(file.name + "LOG_FILE" + file.date).digest("hex")){
try{
if (fs.existsSync(file.name)) {
fs.unlinkSync(file.name);
}
}catch(e){
if (verbose) {
console.error(new Date(), "[FileStreamRotator] Could not remove old log file: ", file.name);
}
}
}
}
/**
* Create symbolic link to current log file
* @param {String} logfile
* @param {String} name Name to use for symbolic link
* @param {Boolean} verbose
*/
function createCurrentSymLink(logfile, name, verbose) {
let symLinkName = name || "current.log"
let logPath = path.dirname(logfile)
let logfileName = path.basename(logfile)
let current = logPath + "/" + symLinkName
try {
let stats = fs.lstatSync(current)
if(stats.isSymbolicLink()){
fs.unlinkSync(current)
fs.symlinkSync(logfileName, current)
}
} catch (err) {
if(err && err.code == "ENOENT") {
try {
fs.symlinkSync(logfileName, current)
} catch (e) {
if (verbose) {
console.error(new Date(), "[FileStreamRotator] Could not create symlink file: ", current, ' -> ', logfileName);
}
}
}
}
}
/**
*
* @param {String} logfile
* @param {Boolean} verbose
* @param {function} cb
*/
function createLogWatcher(logfile, verbose, cb){
if(!logfile) return null
// console.log("Creating log watcher")
try {
let stats = fs.lstatSync(logfile)
return fs.watch(logfile, function(event,filename){
// console.log(Date(), event, filename)
if(event == "rename"){
try {
let stats = fs.lstatSync(logfile)
// console.log("STATS:", stats)
}catch(err){
// console.log("ERROR:", err)
cb(err,logfile)
}
}
})
}catch(err){
if(verbose){
console.log(new Date(),"[FileStreamRotator] Could not add watcher for " + logfile);
}
}
}
/**
* Write audit json object to disk
* @param {String} logfile
* @param {Object} audit
* @param {Object} audit.keep
* @param {Boolean} audit.keep.days
* @param {Number} audit.keep.amount
* @param {String} audit.auditLog
* @param {String} audit.hashType
* @param {Array} audit.files
* @param {EventEmitter} stream
* @param {Boolean} verbose
*/
FileStreamRotator.addLogToAudit = function(logfile, audit, stream, verbose){
if(audit && audit.files){
// Based on contribution by @nickbug - https://github.com/nickbug
var index = audit.files.findIndex(function(file) {
return (file.name === logfile);
});
if (index !== -1) {
// nothing to do as entry already exists.
return audit;
}
var time = Date.now();
audit.files.push({
date: time,
name: logfile,
hash: crypto.createHash(audit.hashType).update(logfile + "LOG_FILE" + time).digest("hex")
});
if(audit.keep.days){
var oldestDate = moment().subtract(audit.keep.amount,"days").valueOf();
var recentFiles = audit.files.filter(function(file){
if(file.date > oldestDate){
return true;
}
file.hashType = audit.hashType
removeFile(file, verbose);
stream.emit("logRemoved", file)
return false;
});
audit.files = recentFiles;
}else{
var filesToKeep = audit.files.splice(-audit.keep.amount);
if(audit.files.length > 0){
audit.files.filter(function(file){
file.hashType = audit.hashType
removeFile(file, verbose);
stream.emit("logRemoved", file)
return false;
})
}
audit.files = filesToKeep;
}
FileStreamRotator.writeAuditLog(audit, verbose);
}
return audit;
}
/**
*
* @param options
* @param options.filename
* @param options.frequency
* @param options.verbose
* @param options.date_format
* @param options.size
* @param options.max_logs
* @param options.audit_file
* @param options.file_options
* @param options.utc
* @param options.extension File extension to be added at the end of the filename
* @param options.watch_log
* @param options.create_symlink
* @param options.symlink_name
* @param options.audit_hash_type Hash to be used to add to the audit log (md5, sha256)
* @returns {Object} stream
*/
FileStreamRotator.getStream = function (options) {
var frequencyMetaData = null;
var curDate = null;
var self = this;
if (!options.filename) {
console.error(new Date(),"[FileStreamRotator] No filename supplied. Defaulting to STDOUT");
return process.stdout;
}
if (options.frequency) {
frequencyMetaData = self.getFrequency(options.frequency);
}
let auditLog = self.setAuditLog(options.max_logs, options.audit_file, options.filename);
// Thanks to Means88 for PR.
if (auditLog != null) {
auditLog.hashType = (options.audit_hash_type !== undefined ? options.audit_hash_type : 'md5');
}
self.verbose = (options.verbose !== undefined ? options.verbose : true);
var fileSize = null;
var fileCount = 0;
var curSize = 0;
if(options.size){
fileSize = FileStreamRotator.parseFileSize(options.size);
}
var dateFormat = (options.date_format || DATE_FORMAT);
if(frequencyMetaData && frequencyMetaData.type == "daily"){
if(!options.date_format){
dateFormat = "YYYY-MM-DD";
}
if(moment().format(dateFormat) != moment().endOf("day").format(dateFormat) || moment().format(dateFormat) == moment().add(1,"day").format(dateFormat)){
if(self.verbose){
console.log(new Date(),"[FileStreamRotator] Changing type to custom as date format changes more often than once a day or not every day");
}
frequencyMetaData.type = "custom";
}
}
if (frequencyMetaData) {
curDate = (options.frequency ? self.getDate(frequencyMetaData,dateFormat, options.utc) : "");
}
options.create_symlink = options.create_symlink || false;
options.extension = options.extension || ""
var filename = options.filename;
var oldFile = null;
var logfile = filename + (curDate ? "." + curDate : "");
if(filename.match(/%DATE%/)){
logfile = filename.replace(/%DATE%/g,(curDate?curDate:self.getDate(null,dateFormat, options.utc)));
}
if(fileSize){
var lastLogFile = null;
var t_log = logfile;
var f = null;
if(auditLog && auditLog.files && auditLog.files instanceof Array && auditLog.files.length > 0){
var lastEntry = auditLog.files[auditLog.files.length - 1].name;
if(lastEntry.match(t_log)){
var lastCount = lastEntry.match(t_log + "\\.(\\d+)");
// Thanks for the PR contribution from @andrefarzat - https://github.com/andrefarzat
if(lastCount){
t_log = lastEntry;
fileCount = lastCount[1];
}
}
}
if (fileCount == 0 && t_log == logfile) {
t_log += options.extension
}
while(f = fs.existsSync(t_log)){
lastLogFile = t_log;
fileCount++;
t_log = logfile + "." + fileCount + options.extension;
}
if(lastLogFile){
var lastLogFileStats = fs.statSync(lastLogFile);
if(lastLogFileStats.size < fileSize){
t_log = lastLogFile;
fileCount--;
curSize = lastLogFileStats.size;
}
}
logfile = t_log;
} else {
logfile += options.extension
}
if (self.verbose) {
console.log(new Date(),"[FileStreamRotator] Logging to: ", logfile);
}
mkDirForFile(logfile);
var file_options = options.file_options || {flags: 'a'};
var rotateStream = fs.createWriteStream(logfile, file_options);
if ((curDate && frequencyMetaData && (staticFrequency.indexOf(frequencyMetaData.type) > -1)) || fileSize > 0) {
if (self.verbose) {
console.log(new Date(),"[FileStreamRotator] Rotating file: ", frequencyMetaData?frequencyMetaData.type:"", fileSize?"size: " + fileSize:"");
}
var stream = new EventEmitter();
stream.auditLog = auditLog;
stream.end = function(){
rotateStream.end.apply(rotateStream,arguments);
};
BubbleEvents(rotateStream,stream);
stream.on('close', function(){
if (logWatcher) {
logWatcher.close()
}
})
stream.on("new",function(newLog){
// console.log("new log", newLog)
stream.auditLog = self.addLogToAudit(newLog,stream.auditLog, stream, self.verbose)
if(options.create_symlink){
createCurrentSymLink(newLog, options.symlink_name, self.verbose)
}
if(options.watch_log){
stream.emit("addWatcher", newLog)
}
});
var logWatcher;
stream.on("addWatcher", function(newLog){
if (logWatcher) {
logWatcher.close()
}
if(!options.watch_log){
return
}
// console.log("ADDING WATCHER", newLog)
logWatcher = createLogWatcher(newLog, self.verbose, function(err,newLog){
stream.emit('createLog', newLog)
})
})
stream.on("createLog",function(file){
try {
let stats = fs.lstatSync(file)
}catch(err){
if(rotateStream && rotateStream.end == "function"){
rotateStream.end();
}
rotateStream = fs.createWriteStream(file, file_options);
stream.emit('new',file);
BubbleEvents(rotateStream,stream);
}
});
stream.write = (function (str, encoding) {
var newDate = frequencyMetaData ? this.getDate(frequencyMetaData, dateFormat, options.utc) : curDate;
if (newDate != curDate || (fileSize && curSize > fileSize)) {
var newLogfile = filename + (curDate && frequencyMetaData ? "." + newDate : "");
if(filename.match(/%DATE%/) && curDate){
newLogfile = filename.replace(/%DATE%/g,newDate);
}
if(fileSize && curSize > fileSize){
fileCount++;
newLogfile += "." + fileCount + options.extension;
}else{
// reset file count
fileCount = 0;
newLogfile += options.extension
}
curSize = 0;
if (self.verbose) {
console.log(new Date(),require('util').format("[FileStreamRotator] Changing logs from %s to %s", logfile, newLogfile));
}
curDate = newDate;
oldFile = logfile;
logfile = newLogfile;
// Thanks to @mattberther https://github.com/mattberther for raising it again.
if(options.end_stream === true){
rotateStream.end();
}else{
rotateStream.destroy();
}
mkDirForFile(logfile);
rotateStream = fs.createWriteStream(newLogfile, file_options);
stream.emit('new',newLogfile);
stream.emit('rotate',oldFile, newLogfile);
BubbleEvents(rotateStream,stream);
}
rotateStream.write(str, encoding);
// Handle length of double-byte characters
curSize += Buffer.byteLength(str, encoding);
}).bind(this);
process.nextTick(function(){
stream.emit('new',logfile);
})
stream.emit('new',logfile)
return stream;
} else {
if (self.verbose) {
console.log(new Date(),"[FileStreamRotator] File won't be rotated: ", options.frequency, options.size);
}
process.nextTick(function(){
rotateStream.emit('new',logfile);
})
return rotateStream;
}
}
/**
* Check and make parent directory
* @param pathWithFile
*/
var mkDirForFile = function(pathWithFile){
var _path = path.dirname(pathWithFile);
_path.split(path.sep).reduce(
function(fullPath, folder) {
fullPath += folder + path.sep;
// Option to replace existsSync as deprecated. Maybe in a future release.
// try{
// var stats = fs.statSync(fullPath);
// console.log('STATS',fullPath, stats);
// }catch(e){
// fs.mkdirSync(fullPath);
// console.log("STATS ERROR",e)
// }
if (!fs.existsSync(fullPath)) {
try{
fs.mkdirSync(fullPath);
}catch(e){
if(e.code !== 'EEXIST'){
throw e;
}
}
}
return fullPath;
},
''
);
};
/**
* Bubbles events to the proxy
* @param emitter
* @param proxy
* @constructor
*/
var BubbleEvents = function BubbleEvents(emitter,proxy){
emitter.on('close',function(){
proxy.emit('close');
})
emitter.on('finish',function(){
proxy.emit('finish');
})
emitter.on('error',function(err){
proxy.emit('error',err);
})
emitter.on('open',function(fd){
proxy.emit('open',fd);
})
}