From fdba23a7eb370edaa314d7fd6bc84a87bc8aed18 Mon Sep 17 00:00:00 2001 From: Shweta Date: Wed, 4 Jan 2017 17:15:12 +0530 Subject: [PATCH 1/3] queue support in case of error during sending logs to loggly -make buffer size configurable -handle 403 error(bad token) -retries to send buffered logs to loggly in every 5 sec -queue support for bulk and input mode --- lib/loggly/client.js | 2 ++ lib/loggly/common.js | 82 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 10 deletions(-) diff --git a/lib/loggly/client.js b/lib/loggly/client.js index 9f18f30..02d411b 100644 --- a/lib/loggly/client.js +++ b/lib/loggly/client.js @@ -56,6 +56,7 @@ var Loggly = exports.Loggly = function (options) { this.userAgent = 'node-loggly ' + loggly.version; this.useTagHeader = 'useTagHeader' in options ? options.useTagHeader : true; this.isBulk = options.isBulk || false; + this.numberOfLogtobebuffered = options.numberOfLogtobebuffered || 1000; // // Set the tags on this instance. // @@ -119,6 +120,7 @@ Loggly.prototype.log = function (msg, tags, callback) { body: msg, proxy: this.proxy, isBulk: this.isBulk, + numberOfLogtobebuffered: this.numberOfLogtobebuffered, headers: { host: this.host, accept: '*/*', diff --git a/lib/loggly/common.js b/lib/loggly/common.js index 785563d..5aba5e1 100644 --- a/lib/loggly/common.js +++ b/lib/loggly/common.js @@ -13,6 +13,17 @@ var arrSize = 100, arrMsg = [], timerFunction = null; +// +// Variables for buffer array +// +var arrBufferedMsg = [], + timerFunctionForBufferedLogs = null; + +// +// flag variable to validate authToken +// +var isValidToken = true; + var https = require('https'), util = require('util'), request = require('request'), @@ -77,7 +88,8 @@ common.loggly = function () { auth, proxy, isBulk, - uri; + uri, + numberOfLogtobebuffered; // // Now that we've popped off the two callbacks @@ -99,6 +111,7 @@ common.loggly = function () { isBulk = args[0].isBulk; headers = args[0].headers; proxy = args[0].proxy; + numberOfLogtobebuffered = args[0].numberOfLogtobebuffered; } } else if (args.length === 2) { @@ -113,6 +126,18 @@ common.loggly = function () { } function onError(err) { + if(!isValidToken){ + console.log(err); + return; + } + var arrayLogs = []; + if(isBulk) { + arrayLogs = requestOptions.body.split('\n'); + } else { + arrayLogs.push(requestOptions.body); + } + storeLogs(arrayLogs); + if (!responded) { responded = true; if (callback) { callback(err) } @@ -138,9 +163,11 @@ common.loggly = function () { return onError(err); } var statusCode = res.statusCode.toString(); + if(statusCode === '403') isValidToken = false; if (Object.keys(failCodes).indexOf(statusCode) !== -1) { return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); - } + } + sendBufferdLogstoLoggly(); success(res, body); }); } @@ -156,29 +183,26 @@ common.loggly = function () { // Join Array Message with new line ('\n') character // requestOptions.body = arrMsg.join('\n'); + arrMsg.length = 0; try { request(requestOptions, function (err, res, body) { if (err) { return onError(err); } var statusCode = res.statusCode.toString(); + if(statusCode === '403') isValidToken = false; if (Object.keys(failCodes).indexOf(statusCode) !== -1) { return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); } + sendBufferdLogstoLoggly(); success(res, body); }); } catch (ex) { onError(ex); } - finally { - // - // Empty the array - // - arrMsg.length = 0; - } } - if (isBulk === true) { + if (isBulk && isValidToken) { if (timerFunction === null) { timerFunction = setInterval(function () { sendBulkLogs(); @@ -189,9 +213,47 @@ common.loggly = function () { sendBulkLogs(); } } - else { + else if(isValidToken) { sendLogs(); } + + // + // retries to send buffered logs to loggly in every 5 sec + // + if (timerFunctionForBufferedLogs === null) { + timerFunctionForBufferedLogs = setInterval(function () { + if (arrBufferedMsg.length) sendBufferdLogstoLoggly(); + }, 5 * 1000); + } + + + function sendBufferdLogstoLoggly() { + var arrayMessage = []; + var bulkModeBunch = arrSize; + var inputModeBunch = 1; + var logsInBunch = isBulk ? bulkModeBunch : inputModeBunch; + for(var startIndex = 0; startIndex < arrBufferedMsg.length; startIndex = startIndex + logsInBunch) { + arrayMessage = arrBufferedMsg.slice(startIndex, startIndex + logsInBunch); + requestOptions.body = isBulk ? arrayMessage.join('\n') : arrayMessage[0]; + request(requestOptions, function (err, res, body) { + if(err) return; + var statusCode = res.statusCode.toString(); + if(statusCode === "200"){ + arrBufferedMsg.splice(0, logsInBunch); + } + }); + } + } + +// +// This function will store logs into buffer +// + function storeLogs(logs) { + if (!logs.length) return; + var numberOfLogsToBeRemoved = (arrBufferedMsg.length + logs.length) - numberOfLogtobebuffered; + if (numberOfLogsToBeRemoved > 0) arrBufferedMsg = arrBufferedMsg.splice(numberOfLogsToBeRemoved); + arrBufferedMsg = arrBufferedMsg.concat(logs); + } }; // // ### function serialize (obj, key) From 584da8503ed74db530ece021a469de081bc407e2 Mon Sep 17 00:00:00 2001 From: Shweta Date: Fri, 6 Jan 2017 18:22:28 +0530 Subject: [PATCH 2/3] Make buffer option configurable and truncate events size over 1 MB --- lib/loggly/client.js | 16 ++++++++++++++-- lib/loggly/common.js | 35 ++++++++++++++++++----------------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/lib/loggly/client.js b/lib/loggly/client.js index 02d411b..e8e4816 100644 --- a/lib/loggly/client.js +++ b/lib/loggly/client.js @@ -23,6 +23,17 @@ function stringify(msg) { return payload; } // +// function to truncate message over 1 MB +// +function truncateLargeMessage(message) { + var MaximumBytesAllowedToLoggly = 1000 * 1000; + var bytesLengthOfLogMessage = Buffer.byteLength(message); + if(bytesLengthOfLogMessage > MaximumBytesAllowedToLoggly) { + message = message.slice(0, MaximumBytesAllowedToLoggly); + } + return message; +} +// // function createClient (options) // Creates a new instance of a Loggly client. // @@ -56,7 +67,7 @@ var Loggly = exports.Loggly = function (options) { this.userAgent = 'node-loggly ' + loggly.version; this.useTagHeader = 'useTagHeader' in options ? options.useTagHeader : true; this.isBulk = options.isBulk || false; - this.numberOfLogtobebuffered = options.numberOfLogtobebuffered || 1000; + this.bufferOptions = options.bufferOptions || {size: 500, retriesInMilliSeconds: 30 * 1000}; // // Set the tags on this instance. // @@ -91,6 +102,7 @@ util.inherits(Loggly, events.EventEmitter); // - http://www.loggly.com/docs/api-sending-data/ // Loggly.prototype.log = function (msg, tags, callback) { + msg.message = truncateLargeMessage(msg.message) if (!callback && typeof tags === 'function') { callback = tags; tags = null; @@ -120,7 +132,7 @@ Loggly.prototype.log = function (msg, tags, callback) { body: msg, proxy: this.proxy, isBulk: this.isBulk, - numberOfLogtobebuffered: this.numberOfLogtobebuffered, + bufferOptions: this.bufferOptions, headers: { host: this.host, accept: '*/*', diff --git a/lib/loggly/common.js b/lib/loggly/common.js index 5aba5e1..85de969 100644 --- a/lib/loggly/common.js +++ b/lib/loggly/common.js @@ -89,7 +89,7 @@ common.loggly = function () { proxy, isBulk, uri, - numberOfLogtobebuffered; + bufferOptions; // // Now that we've popped off the two callbacks @@ -111,7 +111,7 @@ common.loggly = function () { isBulk = args[0].isBulk; headers = args[0].headers; proxy = args[0].proxy; - numberOfLogtobebuffered = args[0].numberOfLogtobebuffered; + bufferOptions = args[0].bufferOptions; } } else if (args.length === 2) { @@ -150,6 +150,8 @@ common.loggly = function () { proxy: proxy }; + var requestOptionsForBufferedLogs = JSON.parse(JSON.stringify(requestOptions)) + if (auth) { requestOptions.headers.authorization = 'Basic ' + new Buffer(auth.username + ':' + auth.password).toString('base64'); } @@ -167,7 +169,6 @@ common.loggly = function () { if (Object.keys(failCodes).indexOf(statusCode) !== -1) { return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); } - sendBufferdLogstoLoggly(); success(res, body); }); } @@ -194,7 +195,6 @@ common.loggly = function () { if (Object.keys(failCodes).indexOf(statusCode) !== -1) { return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode]))); } - sendBufferdLogstoLoggly(); success(res, body); }); } @@ -223,26 +223,27 @@ common.loggly = function () { if (timerFunctionForBufferedLogs === null) { timerFunctionForBufferedLogs = setInterval(function () { if (arrBufferedMsg.length) sendBufferdLogstoLoggly(); - }, 5 * 1000); + }, bufferOptions.retriesInMilliSeconds); } function sendBufferdLogstoLoggly() { + if (!arrBufferedMsg.length) return; var arrayMessage = []; var bulkModeBunch = arrSize; var inputModeBunch = 1; var logsInBunch = isBulk ? bulkModeBunch : inputModeBunch; - for(var startIndex = 0; startIndex < arrBufferedMsg.length; startIndex = startIndex + logsInBunch) { - arrayMessage = arrBufferedMsg.slice(startIndex, startIndex + logsInBunch); - requestOptions.body = isBulk ? arrayMessage.join('\n') : arrayMessage[0]; - request(requestOptions, function (err, res, body) { - if(err) return; - var statusCode = res.statusCode.toString(); - if(statusCode === "200"){ - arrBufferedMsg.splice(0, logsInBunch); - } - }); - } + arrayMessage = arrBufferedMsg.slice(0, logsInBunch); + requestOptionsForBufferedLogs.body = isBulk ? arrayMessage.join('\n') : arrayMessage[0]; + request(requestOptionsForBufferedLogs, function (err, res, body) { + if(err) return; + var statusCode = res.statusCode.toString(); + if(statusCode === "200") { + arrBufferedMsg.splice(0, logsInBunch); + sendBufferdLogstoLoggly(); + } + }); + requestOptionsForBufferedLogs.body = ''; } // @@ -250,7 +251,7 @@ common.loggly = function () { // function storeLogs(logs) { if (!logs.length) return; - var numberOfLogsToBeRemoved = (arrBufferedMsg.length + logs.length) - numberOfLogtobebuffered; + var numberOfLogsToBeRemoved = (arrBufferedMsg.length + logs.length) - bufferOptions.size; if (numberOfLogsToBeRemoved > 0) arrBufferedMsg = arrBufferedMsg.splice(numberOfLogsToBeRemoved); arrBufferedMsg = arrBufferedMsg.concat(logs); } From da058a3336219795f2cedb254df6028e8f1e2627 Mon Sep 17 00:00:00 2001 From: Shweta Date: Wed, 11 Jan 2017 13:57:46 +0530 Subject: [PATCH 3/3] store maximum bytes value into a constant variable --- lib/loggly/client.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/loggly/client.js b/lib/loggly/client.js index e8e4816..d389493 100644 --- a/lib/loggly/client.js +++ b/lib/loggly/client.js @@ -5,7 +5,10 @@ * MIT LICENSE * */ - +// +// Setting constant value of EVENT_SIZE variable +// +var EVENT_SIZE = 1000 * 1000; var events = require('events'), util = require('util'), qs = require('querystring'), @@ -26,7 +29,7 @@ function stringify(msg) { // function to truncate message over 1 MB // function truncateLargeMessage(message) { - var MaximumBytesAllowedToLoggly = 1000 * 1000; + var maximumBytesAllowedToLoggly = EVENT_SIZE; var bytesLengthOfLogMessage = Buffer.byteLength(message); if(bytesLengthOfLogMessage > MaximumBytesAllowedToLoggly) { message = message.slice(0, MaximumBytesAllowedToLoggly);