From a24007eba59c39c8b3f077a594577325133ed2f0 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 24 Jun 2014 18:42:02 +0530 Subject: [PATCH 01/95] Added customer token to configsyslog --- Linux Script/configure-linux.sh | 34 ++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index a36315f..17d8822 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -78,16 +78,15 @@ MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are #this variable is set if the script is invoked via some other calling script IS_INVOKED= +#this variable will hold if the check env function for linux is invoked +LINUX_ENV_VALIDATED= ########## Variable Declarations - End ########## -# executing the script for loggly to install and configure rsyslog. -installLogglyConf() +#check if the Linux environment is compatible with Loggly. +#Also set few variables after the check. +checkLinuxLogglyCompatibility() { - - #log message indicating starting of Loggly configuration - logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." - #check if the user has root permission to run this script checkIfUserHasRootPrivileges @@ -117,11 +116,24 @@ installLogglyConf() #check if selinux service is enforced. if yes, ask the user to manually disable and exit the script checkIfSelinuxServiceEnforced + + LINUX_ENV_VALIDATED="true" +} + +# executing the script for loggly to install and configure rsyslog. +installLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." + + if [ "$LINUX_ENV_VALIDATED" = "" ]; then + checkLinuxLogglyCompatibility + fi #if all the above check passes, write the 22-loggly.conf file write22LogglyConfFile - # Create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu + #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu createRsyslogDir #check if the logs are going to loggly fro linux system now @@ -131,7 +143,6 @@ installLogglyConf() logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." } -# End of configure rsyslog for linux #remove loggly configuration from Linux system removeLogglyConf() @@ -217,7 +228,8 @@ checkIfLogglyServersAccessible() if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then echo "INFO: $LOGGLY_ACCOUNT_URL is reachable." else - logMsgToConfigSysLog "WARNING" "WARNING: $LOGGLY_ACCOUNT_URL is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." + logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_ACCOUNT_URL is not reachable. Please check your network and firewall settings." + exit 1 fi echo "INFO: Checking if $LOGS_01_HOST is reachable." @@ -508,9 +520,9 @@ logMsgToConfigSysLog() sendPayloadToConfigSysLog() { if [ "$APP_TAG" = "" ]; then - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}" + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}" else - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}" + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}" fi curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 } From 7d1019027de004c2c21516135132c6ef6b67fcd7 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 24 Jun 2014 18:42:33 +0530 Subject: [PATCH 02/95] Validated Tomcat installation before Linux --- Modular Scripts/Tomcat/configure-tomcat.sh | 175 ++++++++++++++++----- 1 file changed, 132 insertions(+), 43 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index c1a927a..023f255 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -54,30 +54,40 @@ LOG4J_FILE_PATH= LOGGLY_CATALINA_HOME= MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Tomcat is available at https://www.loggly.com/docs/tomcat-application-server" + +#this variable will hold if the check env function for linux is invoked +TOMCAT_ENV_VALIDATED= ########## Variable Declarations - End ########## +#check if Tomcat environment is compatible for Loggly +checkTomcatLogglyCompatibility() +{ + #check if the linux environment is compatible for Loggly + checkLinuxLogglyCompatibility + + #deduce CATALINA_HOME, this sets the value for LOGGLY_CATALINA_HOME variable + deduceAndCheckTomcatHomeAndVersion + + #check if tomcat is configured with log4j. If yes, then exit + checkIfTomcatConfiguredWithLog4J + + TOMCAT_ENV_VALIDATED="true" +} + + # executing the script for loggly to install and configure syslog. installLogglyConfForTomcat() { - installLogglyConf - #log message indicating starting of Loggly configuration logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Tomcat." - - #get CATALINA_HOME, this sets the value for LOGGLY_CATALINA_HOME variable - getTomcatHome $SERVICE - - #check if the provided or deduced tomcat home is correct or not - checkIfValidTomcatHome - - #set all the required tomcat variables by this script - setTomcatVariables - - #check if tomcat version is supported by the script. The script only support tomcat 6 and 7 - checkIfSupportedTomcatVersion - - #check if tomcat is configured with log4j. If yes, then exit - checkIfTomcatConfiguredWithLog4J + + #check if tomcat environment is compatible with Loggly + if [ "$TOMCAT_ENV_VALIDATED" = "" ]; then + checkTomcatLogglyCompatibility + fi + + #configure loggly for Linux + installLogglyConf #backing up the logging.properties file backupLoggingPropertiesFile @@ -94,27 +104,20 @@ installLogglyConfForTomcat() #log success message logMsgToConfigSysLog "SUCCESS" "SUCCESS: Tomcat successfully configured to send logs via Loggly." } -# End of configure rsyslog for tomcat - +#executing script to remove loggly configuration for tomcat removeLogglyConfForTomcat() { logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." #check if the user has root permission to run this script checkIfUserHasRootPrivileges - + #check if the OS is supported by the script. If no, then exit checkIfSupportedOS - #get CATALINA_HOME, this sets the value for LOGGLY_CATALINA_HOME variable - getTomcatHome $SERVICE - - #check if the provided or deduced tomcat home is correct or not - checkIfValidTomcatHome - - #set all the required tomcat variables by this script - setTomcatVariables + #deduce CATALINA_HOME, this sets the value for LOGGLY_CATALINA_HOME variable + deduceTomcatHomeAndVersion #restore original loggly properties file from backup restoreLogglyPropertiesFile @@ -125,8 +128,91 @@ removeLogglyConfForTomcat() logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } +#identify if tomcat6 or tomcat7 is installed on your system +deduceAndCheckTomcatHomeAndVersion() +{ + + if [ "$LOGGLY_CATALINA_HOME" = "" ]; then + LOGGLY_CATALINA_HOME= + + #lets check if tomcat7 is installed on the system + SERVICE=tomcat7 + + #try to deduce tomcat home considering tomcat7 + assumeTomcatHome $SERVICE + + #initialize validTomcatHome variable with value true. This value will be toggled + #in the function checkIfValidTomcatHome fails + validTomcatHome="true" + + #checks if the deduced tomcat7 home is correct or not + checkIfValidTomcatHome validTomcatHome + + #if tomcat7 home is not valid one, move on to check for tomcat6 + if [ "$validTomcatHome" = "false" ]; then + + LOGGLY_CATALINA_HOME= + + #lets check if tomcat6 is installed on the system + SERVICE=tomcat6 + + #try to deduce tomcat home considering tomcat6 + assumeTomcatHome $SERVICE + + #initialize validTomcatHome variable with value true. This value will be toggled + #in the function checkIfValidTomcatHome fails + validTomcatHome="true" + + #checks if the deduced tomcat7 home is correct or not + checkIfValidTomcatHome validTomcatHome + fi + + if [ "$validTomcatHome" = "true" ]; then + logMsgToConfigSysLog "INFO" "INFO: CATALINA HOME: $LOGGLY_CATALINA_HOME" + + #set all the required tomcat variables by this script + setTomcatVariables + + #find tomcat version + getTomcatVersion + + #check if tomcat version is supported by the script. The script only support tomcat 6 and 7 + checkIfSupportedTomcatVersion + else + logMsgToConfigSysLog "ERROR" "ERROR: Unable to determine correct CATALINA_HOME. Please provide correct Catalina Home using -ch option." + fi + else + #if the user has provided catalina_home, then we need to check if it is a valid catalina home and what is the correct version of the tomcat. + #Let us assume service name is tomcat for now, which will be updated later. + SERVICE=tomcat + checkIfValidTomcatHome validTomcatHome + + if [ "$validTomcatHome" = "true" ]; then + logMsgToConfigSysLog "INFO" "INFO: CATALINA HOME: $LOGGLY_CATALINA_HOME" + + #set tomcat variables + setTomcatVariables + + #find tomcat version + getTomcatVersion + + #check if tomcat version is supported by the script. The script only support tomcat 6 and 7 + checkIfSupportedTomcatVersion + + #update the service name + if ["$tomcatMajorVersion" = "7" ]; then + SERVICE=tomcat7 + elif ["$tomcatMajorVersion" = "6" ]; then + SERVICE=tomcat6 + fi + else + logMsgToConfigSysLog "ERROR" "ERROR: Provided Catalina Home is not correct. Please recheck." + fi + fi +} + #Get default location of tomcat home on various supported OS if user has not provided one -getTomcatHome() +assumeTomcatHome() { #if user has not provided the catalina home if [ "$LOGGLY_CATALINA_HOME" = "" ]; then @@ -142,7 +228,6 @@ getTomcatHome() ;; esac fi - logMsgToConfigSysLog "INFO" "INFO: CATALINA HOME: $LOGGLY_CATALINA_HOME" } #checks if the catalina home is a valid one by searching for logging.properties and @@ -151,14 +236,14 @@ checkIfValidTomcatHome() { #check if logging.properties files is present if [ ! -f "$LOGGLY_CATALINA_HOME/conf/logging.properties" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Unable to find conf/logging.properties file within $LOGGLY_CATALINA_HOME. Please provide correct Catalina Home using -ch option." - exit 1 + logMsgToConfigSysLog "WARN" "WARN: Unable to find conf/logging.properties file within $LOGGLY_CATALINA_HOME." + eval $1="false" #check if tomcat is configured as a service. If no, then check if we have access to startup.sh file elif [ ! -f /etc/init.d/$SERVICE ]; then logMsgToConfigSysLog "INFO" "INFO: Tomcat is not configured as a service" if [ ! -f "$LOGGLY_CATALINA_HOME/bin/startup.sh" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Unable to find bin/startup.sh file within $LOGGLY_CATALINA_HOME. Please provide correct Catalina Home using -ch option." - exit 1 + logMsgToConfigSysLog "WARN" "WARN: Unable to find bin/startup.sh file within $LOGGLY_CATALINA_HOME." + eval $1="false" fi fi } @@ -178,9 +263,8 @@ setTomcatVariables() CATALINA_JAR_PATH=$LOGGLY_CATALINA_HOME/lib/catalina.jar } -#checks if the tomcat version is supported by this script, currently the script -#only supports tomcat 6 and tomcat 7 -checkIfSupportedTomcatVersion() +#get the version of tomcat +getTomcatVersion() { #check if the identified CATALINA_HOME has the catalina.jar if [ ! -f "$CATALINA_JAR_PATH" ]; then @@ -202,12 +286,17 @@ checkIfSupportedTomcatVersion() TOMCAT_VERSION=${TOMCAT_VERSION#*: } TOMCAT_VERSION=$TOMCAT_VERSION | tr -d ' ' APP_TAG="\"tomcat-version\":\"$TOMCAT_VERSION\"" + fi +} - tomcatMajorVersion=${TOMCAT_VERSION%%.*} - if [[ ($tomcatMajorVersion -ne 6 ) && ($tomcatMajorVersion -ne 7) ]]; then - echo "ERROR" "ERROR: This script only supports Tomcat version 6 or 7." - exit 1 - fi +#checks if the tomcat version is supported by this script, currently the script +#only supports tomcat 6 and tomcat 7 +checkIfSupportedTomcatVersion() +{ + tomcatMajorVersion=${TOMCAT_VERSION%%.*} + if [[ ($tomcatMajorVersion -ne 6 ) && ($tomcatMajorVersion -ne 7) ]]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script only supports Tomcat version 6 or 7." + exit 1 fi } @@ -330,7 +419,7 @@ write21TomcatFileContents() logMsgToConfigSysLog "INFO" "INFO: Creating file $TOMCAT_SYSLOG_CONFFILE" sudo touch $TOMCAT_SYSLOG_CONFFILE sudo chmod o+w $TOMCAT_SYSLOG_CONFFILE - + imfileStr="\$ModLoad imfile \$WorkDirectory $SYSLOG_DIR " From b12a1f154c4261b167600f0f9e03675326ec38b8 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 26 Jun 2014 23:37:08 +0530 Subject: [PATCH 03/95] Added support to check Loggly port reachabilty Also modified script to Log and Exit if any of the Loggly server is not accessible. --- Linux Script/configure-linux.sh | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 17d8822..8dfcf21 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -236,14 +236,24 @@ checkIfLogglyServersAccessible() if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then echo "INFO: $LOGS_01_HOST is reachable." else - logMsgToConfigSysLog "WARNING" "WARNING: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." + exit 1 + fi + + echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." + if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then + echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system." + exit 1 fi - echo "INFO: Checking if Gen2 account" + echo "INFO: Checking if Gen2 account." if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "404 NOT FOUND" | wc -l) == 1 ]; then logMsgToConfigSysLog "ERROR" "ERROR: This scripts need a Gen2 account. Please contact Loggly support." + exit 1 else - echo "INFO: It is a Gen2 account" + echo "INFO: It is a Gen2 account." fi } @@ -293,6 +303,7 @@ checkIfMultipleRsyslogConfigured() { if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." + exit 1 fi } @@ -317,6 +328,7 @@ checkIfSelinuxServiceEnforced() logMsgToConfigSysLog "INFO" "INFO: selinux status is not enforced." elif [ $(sudo getenforce | grep "Enforcing" | wc -l) -gt 0 ]; then logMsgToConfigSysLog "ERROR" "ERROR: selinux status is 'Enforcing'. Please disable it and start the rsyslog daemon manually." + exit 1 fi } From 4160bcaa06d0b69e62b5592095b9fab91c995bfd Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 26 Jun 2014 23:37:39 +0530 Subject: [PATCH 04/95] Support for Tomcat downloaded as a package --- Modular Scripts/Tomcat/configure-tomcat.sh | 38 +++++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 023f255..6f80596 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -25,11 +25,6 @@ TOMCAT_SYSLOG_CONFFILE_BACKUP=$SYSLOG_ETCDIR_CONF/21-tomcat.conf.loggly.bk #syslog directory SYSLOG_DIR=/var/spool/rsyslog -#this variable will hold the host name -HOST_NAME= -#this variable will hold the name of the linux distribution -LINUX_DIST= - #this variable will hold the path to the catalina home LOGGLY_CATALINA_HOME= #this variable will hold the path to the conf folder within catalina home @@ -117,7 +112,7 @@ removeLogglyConfForTomcat() checkIfSupportedOS #deduce CATALINA_HOME, this sets the value for LOGGLY_CATALINA_HOME variable - deduceTomcatHomeAndVersion + deduceAndCheckTomcatHomeAndVersion #restore original loggly properties file from backup restoreLogglyPropertiesFile @@ -185,6 +180,11 @@ deduceAndCheckTomcatHomeAndVersion() #if the user has provided catalina_home, then we need to check if it is a valid catalina home and what is the correct version of the tomcat. #Let us assume service name is tomcat for now, which will be updated later. SERVICE=tomcat + + #set the flag to true + validTomcatHome="true" + + #check if the tomcat home provided by user is valid checkIfValidTomcatHome validTomcatHome if [ "$validTomcatHome" = "true" ]; then @@ -200,11 +200,11 @@ deduceAndCheckTomcatHomeAndVersion() checkIfSupportedTomcatVersion #update the service name - if ["$tomcatMajorVersion" = "7" ]; then + if [ "$tomcatMajorVersion" = "7" ]; then SERVICE=tomcat7 - elif ["$tomcatMajorVersion" = "6" ]; then + elif [ "$tomcatMajorVersion" = "6" ]; then SERVICE=tomcat6 - fi + fi else logMsgToConfigSysLog "ERROR" "ERROR: Provided Catalina Home is not correct. Please recheck." fi @@ -258,6 +258,11 @@ setTomcatVariables() LOGGLY_CATALINA_BACKUP_PROPFILE=$LOGGLY_CATALINA_PROPFILE.loggly.bk LOGGLY_CATALINA_LOG_HOME=/var/log/$SERVICE + + #if tomcat is not installed as service, then tomcat logs will be created at would be $CATALINA_HOME/log + if [ ! -f "$LOGGLY_CATALINA_LOG_HOME" ]; then + LOGGLY_CATALINA_LOG_HOME=$LOGGLY_CATALINA_HOME/logs + fi #default path for catalina.jar CATALINA_JAR_PATH=$LOGGLY_CATALINA_HOME/lib/catalina.jar @@ -504,6 +509,9 @@ write21TomcatFileContents() sudo cat << EOIPFW >> $TOMCAT_SYSLOG_CONFFILE $imfileStr EOIPFW + + # restart the syslog service. + restartRsyslog } #checks if the tomcat logs made to loggly @@ -523,9 +531,7 @@ checkIfTomcatLogsMadeToLoggly() #get the initial count of tomcat logs for past 15 minutes searchAndFetch tomcatInitialLogCount "$queryUrl" - logMsgToConfigSysLog "INFO" "INFO: Restarting rsyslog and tomcat to generate logs for verification." - # restart the syslog service. - restartRsyslog + logMsgToConfigSysLog "INFO" "INFO: Restarting tomcat to generate logs for verification." # restart the tomcat service. restartTomcat @@ -615,9 +621,9 @@ restartTomcat() usage() { cat << EOF -usage: ltomcatsetup [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-ch catalina home (optional)] -usage: ltomcatsetup [-r to rollback] [-ch catalina home (optional)] -usage: ltomcatsetup [-h for help] +usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-ch catalina home (optional)] +usage: configure-tomcat [-r to rollback] [-a loggly auth account or subdomain] [-ch catalina home (optional)] +usage: configure-tomcat [-h for help] EOF } @@ -670,7 +676,7 @@ elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAM getPassword fi installLogglyConfForTomcat -elif [ "$LOGGLY_ROLLBACK" != "" ]; then +elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then removeLogglyConfForTomcat else usage From 8e46033abd9e7e91532439da76c89733698c251e Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 30 Jun 2014 19:43:50 +0530 Subject: [PATCH 05/95] Added Apache2 Configuration Script --- Modular Scripts/Apache2/configure-apache.sh | 377 ++++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100644 Modular Scripts/Apache2/configure-apache.sh diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh new file mode 100644 index 0000000..bfa7460 --- /dev/null +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -0,0 +1,377 @@ +#!/bin/bash + +source configure-linux.sh "being-invoked" + +########## Variable Declarations - Start ########## +#name of the current script +SCRIPT_NAME=configure-apache.sh +#version of the current script +SCRIPT_VERSION=1.0 + +#we have not found the apache version yet at this point in the script +APP_TAG="\"apache-version\":\"\"" + +#name of the service, in this case apache2 +SERVICE= +#directory location for syslog +SYSLOG_ETCDIR_CONF=/etc/rsyslog.d +#name and location of apache syslog file +APACHE_SYSLOG_CONFFILE=$SYSLOG_ETCDIR_CONF/21-apache.conf +#name and location of apache syslog backup file +APACHE_SYSLOG_CONFFILE_BACKUP=$SYSLOG_ETCDIR_CONF/21-apache.conf.loggly.bk +#syslog directory +SYSLOG_DIR=/var/spool/rsyslog + +#this variable will hold the path to the apache home +LOGGLY_APACHE_HOME= +#this variable will hold the value of the apache log folder +LOGGLY_APACHE_LOG_HOME= +#this variable will hold the users apache version +APACHE_VERSION= + +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/apache-application-server" + +#this variable will hold if the check env function for linux is invoked +APACHE_ENV_VALIDATED= +########## Variable Declarations - End ########## + +#check if apache environment is compatible for Loggly +checkApacheLogglyCompatibility() +{ + #check if the linux environment is compatible for Loggly + checkLinuxLogglyCompatibility + + #check if apache2 is installed on unix system + checkApacheDetails + + APACHE_ENV_VALIDATED="true" +} + + +# executing the script for loggly to install and configure syslog. +installLogglyConfForApache() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Apache." + + #check if apache environment is compatible with Loggly + if [ "$APACHE_ENV_VALIDATED" = "" ]; then + checkApacheLogglyCompatibility + fi + + #configure loggly for Linux + installLogglyConf + + #check for the apache log file size + checkLogFileSize $LOGGLY_APACHE_LOG_HOME/access.log $LOGGLY_APACHE_LOG_HOME/error.log + + #create 21apache.conf file + write21ApacheConfFile + + #verify if the apache logs made it to loggly + checkIfApacheLogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache successfully configured to send logs via Loggly." +} + +#executing script to remove loggly configuration for Apache +removeLogglyConfForApache() +{ + logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #check if apache2 is installed on unix system + checkApacheDetails + + #remove 21apache.conf file + remove21ApacheConfFile + + logMsgToConfigSysLog "INFO" "INFO: Rollback completed." +} + +#identify if apache2 is installed on your system and is available as a service +checkApacheDetails() +{ + getApacheServiceName + + #verify if apache is installed as service + if [ ! -f /etc/init.d/$SERVICE ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Apache is not configured as a service" + exit 1 + fi + + #get the version of apache installed + getApacheVersion + + #check if apache is supported + checkIfSupportedApacheVersion + + #set all the required apache variables by this script + setApacheVariables +} + +#Get the apache service name on various linux flavors +getApacheServiceName() +{ + case "$LINUX_DIST" in + *"Ubuntu"* ) + SERVICE="apache2" + ;; + *"Red Hat"* ) + SERVICE="httpd" + ;; + *"CentOS"* ) + SERVICE="httpd" + ;; + esac +} + +#sets apache variables which will be used across various functions +setApacheVariables() +{ + LOGGLY_APACHE_LOG_HOME=/var/log/$SERVICE +} + +#gets the version of apache installed on the unix box +getApacheVersion() +{ + APACHE_VERSION=$($SERVICE -v | grep "Server version: Apache") + APACHE_VERSION=${APACHE_VERSION#*/} + APACHE_VERSION=${APACHE_VERSION% *} + APACHE_VERSION=$APACHE_VERSION | tr -d ' ' + APP_TAG="\"apache-version\":\"$APACHE_VERSION\"" + echo "Apache version: " $APACHE_VERSION +} + +#checks if the apache version is supported by this script, currently the script +#only supports apache2 +checkIfSupportedApacheVersion() +{ + apacheMajorVersion=${APACHE_VERSION%%.*} + if [[ ($apacheMajorVersion -ne 2 ) ]]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script only supports Apache version 2." + exit 1 + fi +} + +checkLogFileSize() +{ + accessFileSize=$(wc -c "$1" | cut -f 1 -d ' ') + errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') + fileSize=$((accessFileSize+errorFileSize)) + if [ $fileSize -ge 100000000 ]; then + logMsgToConfigSysLog "INFO" "INFO: " + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize. Continuing with Apache Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize. Discontinuing with Apache Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + elif [ $fileSize -eq 0 ]; then + logMsgToConfigSysLog "WARN" "WARN: There are no recent log files from Apache so verification may not succeed." + fi +} + +write21ApacheConfFile() +{ + #Create apache syslog config file if it doesn't exist + echo "INFO: Checking if apache sysconf file $APACHE_SYSLOG_CONFFILE exist." + if [ -f "$APACHE_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: Apache syslog file $APACHE_SYSLOG_CONFFILE already exist." + while true; do + read -p "Do you wish to override $APACHE_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $APACHE_SYSLOG_CONFFILE to $APACHE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $APACHE_SYSLOG_CONFFILE $APACHE_SYSLOG_CONFFILE_BACKUP; + write21ApacheFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + write21ApacheFileContents + fi +} + +#function to write the contents of apache syslog config file +write21ApacheFileContents() +{ + + logMsgToConfigSysLog "INFO" "INFO: Creating file $APACHE_SYSLOG_CONFFILE" + sudo touch $APACHE_SYSLOG_CONFFILE + sudo chmod o+w $APACHE_SYSLOG_CONFFILE + + imfileStr="\$ModLoad imfile + \$InputFilePollInterval 10 + \$WorkDirectory $SYSLOG_DIR + " + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + imfileStr+="\$PrivDropToGroup adm + " + fi + + imfileStr+=" + # Apache access file: + \$InputFileName $LOGGLY_APACHE_LOG_HOME/access.log + \$InputFileTag apache-access: + \$InputFileStateFile stat-apache-access + \$InputFileSeverity info + \$InputFilePersistStateInterval 20000 + \$InputRunFileMonitor + + #Apache Error file: + \$InputFileName $LOGGLY_APACHE_LOG_HOME/error.log + \$InputFileTag apache-error: + \$InputFileStateFile stat-apache-error + \$InputFileSeverity error + \$InputFilePersistStateInterval 20000 + \$InputRunFileMonitor + + #Add a tag for apache events + \$template LogglyFormatApache,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"apache\\\"] %msg%\n\" + + if \$programname == 'apache-access' then @@logs-01.loggly.com:514;LogglyFormatApache + if \$programname == 'apache-access' then ~ + if \$programname == 'apache-error' then @@logs-01.loggly.com:514;LogglyFormatApache + if \$programname == 'apache-error' then ~ + " + + #change the apache-21 file to variable from above and also take the directory of the apache log file. +sudo cat << EOIPFW >> $APACHE_SYSLOG_CONFFILE +$imfileStr +EOIPFW + + restartRsyslog +} + + +#checks if the apache logs made to loggly +checkIfApacheLogsMadeToLoggly() +{ + counter=1 + maxCounter=10 + + apacheInitialLogCount=0 + apacheLatestLogCount=0 + queryParam="tag%3Aapache&from=-15m&until=now&size=1" + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Getting initial apache log count." + #get the initial count of apache logs for past 15 minutes + searchAndFetch apacheInitialLogCount "$queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the apache logs made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + #get the final count of apache logs for past 15 minutes + searchAndFetch apacheLatestLogCount "$queryUrl" + let counter=$counter+1 + + while [ "$apacheLatestLogCount" -le "$apacheInitialLogCount" ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch apacheLatestLogCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + exit 1 + fi + done + + if [ "$apacheLatestLogCount" -gt "$apacheInitialLogCount" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Apache logs successfully transferred to Loggly! You are now sending Apache logs to Loggly." + exit 0 + fi +} + +#remove 21apache.conf file +remove21ApacheConfFile() +{ + echo "INFO: Deleting the loggly apache syslog conf file." + if [ -f "$APACHE_SYSLOG_CONFFILE" ]; then + sudo rm -rf "$APACHE_SYSLOG_CONFFILE" + fi + echo "INFO: Removed all the modified files." + restartRsyslog +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-ah apache home (optional)] +usage: configure-linux [-a loggly auth account or subdomain] [-r to rollback] +usage: configure-apache [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## + +if [ $# -eq 0 ]; then + usage + exit +else +while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --rollback ) + LOGGLY_ROLLBACK="true" + ;; + -h | --help) + usage + exit + ;; + esac + shift +done +fi + +if [ "$LOGGLY_DEBUG" != "" -a "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + debug +elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConfForApache +elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then + removeLogglyConfForApache +else + usage +fi + +########## Get Inputs from User - End ########## \ No newline at end of file From 040fcaf75f02e20b1e66d3f58f3a1542b4b05e81 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 1 Jul 2014 22:44:07 +0530 Subject: [PATCH 06/95] Download Linux script inside it Also asks user if he wants to restart tomcat --- Modular Scripts/Tomcat/configure-tomcat.sh | 77 +++++++++++++--------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 6f80596..f0d6e90 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -1,5 +1,8 @@ #!/bin/bash +#downloads configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh" +curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh source configure-linux.sh "being-invoked" ########## Variable Declarations - Start ########## @@ -531,7 +534,7 @@ checkIfTomcatLogsMadeToLoggly() #get the initial count of tomcat logs for past 15 minutes searchAndFetch tomcatInitialLogCount "$queryUrl" - logMsgToConfigSysLog "INFO" "INFO: Restarting tomcat to generate logs for verification." + logMsgToConfigSysLog "INFO" "INFO: Tomcat needs to be restarted to complete the configuration and verification." # restart the tomcat service. restartTomcat @@ -579,42 +582,56 @@ remove21TomcatConfFile() sudo rm -rf "$TOMCAT_SYSLOG_CONFFILE" fi echo "INFO: Removed all the modified files." + + logMsgToConfigSysLog "INFO" "INFO: Tomcat needs to be restarted to rollback the configuration." restartTomcat } #restart tomcat restartTomcat() { - #sudo service tomcat restart or home/bin/start.sh - if [ $(ps -ef | grep -v grep | grep "$SERVICE" | wc -l) -gt 0 ]; then - logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running." - if [ -f /etc/init.d/$SERVICE ]; then - logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running as service." - logMsgToConfigSysLog "INFO" "INFO: Restarting the tomcat service." - sudo service $SERVICE restart - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not restart gracefully. Log rotation may not be disabled. Please restart tomcat manually." - fi - else - logMsgToConfigSysLog "INFO" "INFO: $SERVICE is not running as service." - # To be commented only for test - logMsgToConfigSysLog "INFO" "INFO: Shutting down tomcat." - sudo $LOGGLY_CATALINA_HOME/bin/shutdown.sh - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not shut down gracefully." - else - logMsgToConfigSysLog "INFO" "INFO: Done shutting down tomcat." - fi - - logMsgToConfigSysLog "INFO" "INFO: Starting up tomcat." - sudo $LOGGLY_CATALINA_HOME/bin/startup.sh - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not start up down gracefully." - else - logMsgToConfigSysLog "INFO" "INFO: Tomcat is up and running." + while true; do + read -p "Do you wish to restart tomcat server? (yes/no)" yn + case $yn in + [Yy]* ) + #sudo service tomcat restart or home/bin/start.sh + if [ $(ps -ef | grep -v grep | grep "$SERVICE" | wc -l) -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running." + if [ -f /etc/init.d/$SERVICE ]; then + logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running as service." + logMsgToConfigSysLog "INFO" "INFO: Restarting the tomcat service." + sudo service $SERVICE restart + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not restart gracefully. Log rotation may not be disabled. Please restart tomcat manually." + fi + else + logMsgToConfigSysLog "INFO" "INFO: $SERVICE is not running as service." + # To be commented only for test + logMsgToConfigSysLog "INFO" "INFO: Shutting down tomcat." + sudo $LOGGLY_CATALINA_HOME/bin/shutdown.sh + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not shut down gracefully." + else + logMsgToConfigSysLog "INFO" "INFO: Done shutting down tomcat." + fi + + logMsgToConfigSysLog "INFO" "INFO: Starting up tomcat." + sudo $LOGGLY_CATALINA_HOME/bin/startup.sh + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not start up down gracefully." + else + logMsgToConfigSysLog "INFO" "INFO: Tomcat is up and running." + fi + fi fi - fi - fi + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Exiting the script based on your input. Please restart Tomcat manually." + break;; + * ) echo "Please answer yes or no.";; + esac + done + } #display usage syntax From 3f133cd2b767bcbde67065634cdde3657cc532f8 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 1 Jul 2014 22:44:59 +0530 Subject: [PATCH 07/95] Downloads Linux script inside it --- Modular Scripts/Apache2/configure-apache.sh | 27 ++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index bfa7460..04293e1 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -1,7 +1,10 @@ #!/bin/bash +#downloads configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh" +curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh source configure-linux.sh "being-invoked" - + ########## Variable Declarations - Start ########## #name of the current script SCRIPT_NAME=configure-apache.sh @@ -13,6 +16,10 @@ APP_TAG="\"apache-version\":\"\"" #name of the service, in this case apache2 SERVICE= +#name of apache access log file +APACHE_ACCESS_LOG_FILE= +#name of apache error log file +APACHE_ERROR_LOG_FILE= #directory location for syslog SYSLOG_ETCDIR_CONF=/etc/rsyslog.d #name and location of apache syslog file @@ -63,7 +70,7 @@ installLogglyConfForApache() installLogglyConf #check for the apache log file size - checkLogFileSize $LOGGLY_APACHE_LOG_HOME/access.log $LOGGLY_APACHE_LOG_HOME/error.log + checkLogFileSize $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE #create 21apache.conf file write21ApacheConfFile @@ -122,12 +129,18 @@ getApacheServiceName() case "$LINUX_DIST" in *"Ubuntu"* ) SERVICE="apache2" + APACHE_ACCESS_LOG_FILE="access.log" + ERROR_ACCESS_LOG_FILE="error.log" ;; *"Red Hat"* ) SERVICE="httpd" + APACHE_ACCESS_LOG_FILE="access_log" + ERROR_ACCESS_LOG_FILE="error_log" ;; *"CentOS"* ) SERVICE="httpd" + APACHE_ACCESS_LOG_FILE="access_log" + ERROR_ACCESS_LOG_FILE="error_log" ;; esac } @@ -227,7 +240,7 @@ write21ApacheFileContents() imfileStr+=" # Apache access file: - \$InputFileName $LOGGLY_APACHE_LOG_HOME/access.log + \$InputFileName $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE \$InputFileTag apache-access: \$InputFileStateFile stat-apache-access \$InputFileSeverity info @@ -235,7 +248,7 @@ write21ApacheFileContents() \$InputRunFileMonitor #Apache Error file: - \$InputFileName $LOGGLY_APACHE_LOG_HOME/error.log + \$InputFileName $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE \$InputFileTag apache-error: \$InputFileStateFile stat-apache-error \$InputFileSeverity error @@ -341,7 +354,7 @@ while [ "$1" != "" ]; do ;; -u | --username ) shift LOGGLY_USERNAME=$1 - echo "Username is set" + echo "Userna.loggly.com isme is set" ;; -p | --password ) shift LOGGLY_PASSWORD=$1 @@ -360,11 +373,13 @@ fi if [ "$LOGGLY_DEBUG" != "" -a "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then - getPassword + downloadConfigureLinuxScript + downloadConfigureLinuxScriptgetPassword fi debug elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then + downloadConfigureLinuxScript getPassword fi installLogglyConfForApache From 1e00b42ae3f85aae0c404a7f02100d64e7a4939f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 3 Jul 2014 23:24:59 +0530 Subject: [PATCH 08/95] Logging OS name to Loggly if unsupported --- Linux Script/configure-linux.sh | 72 +++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 8dfcf21..97e8377 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -8,7 +8,7 @@ trap ctrl_c INT function ctrl_c() { logMsgToConfigSysLog "INFO" "INFO: Aborting the script." exit 1 -} +} ########## Variable Declarations - Start ########## @@ -116,7 +116,7 @@ checkLinuxLogglyCompatibility() #check if selinux service is enforced. if yes, ask the user to manually disable and exit the script checkIfSelinuxServiceEnforced - + LINUX_ENV_VALIDATED="true" } @@ -161,7 +161,7 @@ removeLogglyConf() #remove 22-loggly.conf file remove22LogglyConfFile - + #restart rsyslog service restartRsyslog @@ -182,31 +182,43 @@ checkIfUserHasRootPrivileges() #check if supported operating system checkIfSupportedOS() { - #set value for linux distribution name - LINUX_DIST=$(lsb_release -ds) - - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: This operating system is not supported by the script." + getOs + + case "$LINUX_DIST" in + *"Ubuntu"* ) + echo "INFO: Operating system is Ubuntu." + ;; + *"Red Hat"* ) + echo "INFO: Operating system is Red Hat." + ;; + *"CentOS"* ) + echo "INFO: Operating system is CentOS." + ;; + * ) + logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." exit 1 - else - #remove double quotes (if any) from the linux distribution name - LINUX_DIST="${LINUX_DIST%\"}" - LINUX_DIST="${LINUX_DIST#\"}" - case "$LINUX_DIST" in - *"Ubuntu"* ) - echo "INFO: Operating system is Ubuntu." - ;; - *"Red Hat"* ) - echo "INFO: Operating system is Red Hat." - ;; - *"CentOS"* ) - echo "INFO: Operating system is CentOS." - ;; - * ) - logMsgToConfigSysLog "ERROR" "ERROR: This operating system is not supported by the script." - exit 1 - ;; - esac + ;; + esac +} + +getOs() +{ + # Determine OS platform + UNAME=$(uname | tr "[:upper:]" "[:lower:]") + # If Linux, try to determine specific distribution + if [ "$UNAME" == "linux" ]; then + # If available, use LSB to identify distribution + if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then + LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) + # Otherwise, use release info file + else + LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) + fi + fi + + # For everything else (or if above failed), just use generic identifier + if [ "$LINUX_DIST" == "" ]; then + LINUX_DIST=$(uname) fi } @@ -239,7 +251,7 @@ checkIfLogglyServersAccessible() logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." exit 1 fi - + echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." @@ -247,7 +259,7 @@ checkIfLogglyServersAccessible() logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system." exit 1 fi - + echo "INFO: Checking if Gen2 account." if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "404 NOT FOUND" | wc -l) == 1 ]; then logMsgToConfigSysLog "ERROR" "ERROR: This scripts need a Gen2 account. Please contact Loggly support." @@ -631,7 +643,7 @@ if [ "$1" != "being-invoked" ]; then fi if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then - removeLogglyConf + removeLogglyConf elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword From c6b391fd548d1c80536c60783899c5759a45d6a4 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 3 Jul 2014 23:45:50 +0530 Subject: [PATCH 09/95] Updated Red Hat to RedHat --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 97e8377..97c11ac 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -188,7 +188,7 @@ checkIfSupportedOS() *"Ubuntu"* ) echo "INFO: Operating system is Ubuntu." ;; - *"Red Hat"* ) + *"RedHat"* ) echo "INFO: Operating system is Red Hat." ;; *"CentOS"* ) From 1de793f6c3e81b814fb7aea2a7afd1e2a47c9deb Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 4 Jul 2014 00:01:49 +0530 Subject: [PATCH 10/95] Updated RedhHat/ CentOS log file name --- Modular Scripts/Apache2/configure-apache.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 04293e1..4e9c0e7 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -130,17 +130,17 @@ getApacheServiceName() *"Ubuntu"* ) SERVICE="apache2" APACHE_ACCESS_LOG_FILE="access.log" - ERROR_ACCESS_LOG_FILE="error.log" + APACHE_ERROR_LOG_FILE="error.log" ;; - *"Red Hat"* ) + *"RedHat"* ) SERVICE="httpd" APACHE_ACCESS_LOG_FILE="access_log" - ERROR_ACCESS_LOG_FILE="error_log" + APACHE_ERROR_LOG_FILE="error_log" ;; *"CentOS"* ) SERVICE="httpd" APACHE_ACCESS_LOG_FILE="access_log" - ERROR_ACCESS_LOG_FILE="error_log" + APACHE_ERROR_LOG_FILE="error_log" ;; esac } @@ -224,7 +224,6 @@ write21ApacheConfFile() #function to write the contents of apache syslog config file write21ApacheFileContents() { - logMsgToConfigSysLog "INFO" "INFO: Creating file $APACHE_SYSLOG_CONFFILE" sudo touch $APACHE_SYSLOG_CONFFILE sudo chmod o+w $APACHE_SYSLOG_CONFFILE @@ -373,13 +372,11 @@ fi if [ "$LOGGLY_DEBUG" != "" -a "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then - downloadConfigureLinuxScript - downloadConfigureLinuxScriptgetPassword + getPassword fi debug elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then - downloadConfigureLinuxScript getPassword fi installLogglyConfForApache From 318770b60dfc1a6f41beefa51b7cf7631b2ca3c8 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 4 Jul 2014 00:05:05 +0530 Subject: [PATCH 11/95] Stop of the user do not want to restart tomcat --- Modular Scripts/Tomcat/configure-tomcat.sh | 123 +++++++++++---------- 1 file changed, 66 insertions(+), 57 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index f0d6e90..cdf6e44 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -62,13 +62,13 @@ checkTomcatLogglyCompatibility() { #check if the linux environment is compatible for Loggly checkLinuxLogglyCompatibility - + #deduce CATALINA_HOME, this sets the value for LOGGLY_CATALINA_HOME variable deduceAndCheckTomcatHomeAndVersion #check if tomcat is configured with log4j. If yes, then exit checkIfTomcatConfiguredWithLog4J - + TOMCAT_ENV_VALIDATED="true" } @@ -78,12 +78,15 @@ installLogglyConfForTomcat() { #log message indicating starting of Loggly configuration logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Tomcat." - + #check if tomcat environment is compatible with Loggly if [ "$TOMCAT_ENV_VALIDATED" = "" ]; then checkTomcatLogglyCompatibility fi - + + #ask user if tomcat can be restarted + canTomcatBeRestarted + #configure loggly for Linux installLogglyConf @@ -117,11 +120,14 @@ removeLogglyConfForTomcat() #deduce CATALINA_HOME, this sets the value for LOGGLY_CATALINA_HOME variable deduceAndCheckTomcatHomeAndVersion - #restore original loggly properties file from backup - restoreLogglyPropertiesFile + #ask user if tomcat can be restarted + canTomcatBeRestarted #remove 21tomcat.conf file remove21TomcatConfFile + + #restore original loggly properties file from backup + restoreLogglyPropertiesFile logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } @@ -183,10 +189,10 @@ deduceAndCheckTomcatHomeAndVersion() #if the user has provided catalina_home, then we need to check if it is a valid catalina home and what is the correct version of the tomcat. #Let us assume service name is tomcat for now, which will be updated later. SERVICE=tomcat - + #set the flag to true validTomcatHome="true" - + #check if the tomcat home provided by user is valid checkIfValidTomcatHome validTomcatHome @@ -207,7 +213,7 @@ deduceAndCheckTomcatHomeAndVersion() SERVICE=tomcat7 elif [ "$tomcatMajorVersion" = "6" ]; then SERVICE=tomcat6 - fi + fi else logMsgToConfigSysLog "ERROR" "ERROR: Provided Catalina Home is not correct. Please recheck." fi @@ -223,7 +229,7 @@ assumeTomcatHome() *"Ubuntu"* ) LOGGLY_CATALINA_HOME="/var/lib/$1" ;; - *"Red Hat"* ) + *"RedHat"* ) LOGGLY_CATALINA_HOME="/usr/share/$1" ;; *"CentOS"* ) @@ -261,7 +267,7 @@ setTomcatVariables() LOGGLY_CATALINA_BACKUP_PROPFILE=$LOGGLY_CATALINA_PROPFILE.loggly.bk LOGGLY_CATALINA_LOG_HOME=/var/log/$SERVICE - + #if tomcat is not installed as service, then tomcat logs will be created at would be $CATALINA_HOME/log if [ ! -f "$LOGGLY_CATALINA_LOG_HOME" ]; then LOGGLY_CATALINA_LOG_HOME=$LOGGLY_CATALINA_HOME/logs @@ -330,6 +336,21 @@ checkIfTomcatConfiguredWithLog4J() logMsgToConfigSysLog "INFO" "INFO: Tomcat seems not to be configured with log4j logger." } +canTomcatBeRestarted() +{ + while true; do + read -p "Tomcat needs to be restarted during configuration. Do you wish to continue? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done +} #backup the logging.properties file in the CATALINA_HOME folder backupLoggingPropertiesFile() { @@ -423,7 +444,6 @@ write21TomcatConfFile() #function to write the contents of tomcat syslog config file write21TomcatFileContents() { - logMsgToConfigSysLog "INFO" "INFO: Creating file $TOMCAT_SYSLOG_CONFFILE" sudo touch $TOMCAT_SYSLOG_CONFFILE sudo chmod o+w $TOMCAT_SYSLOG_CONFFILE @@ -513,7 +533,7 @@ sudo cat << EOIPFW >> $TOMCAT_SYSLOG_CONFFILE $imfileStr EOIPFW - # restart the syslog service. + #restart the syslog service. restartRsyslog } @@ -572,6 +592,9 @@ restoreLogglyPropertiesFile() sudo cp -f $LOGGLY_CATALINA_BACKUP_PROPFILE $LOGGLY_CATALINA_PROPFILE sudo rm -fr $LOGGLY_CATALINA_BACKUP_PROPFILE fi + + logMsgToConfigSysLog "INFO" "INFO: Tomcat needs to be restarted to rollback the configuration." + restartTomcat } #remove 21tomcat.conf file @@ -581,57 +604,43 @@ remove21TomcatConfFile() if [ -f "$TOMCAT_SYSLOG_CONFFILE" ]; then sudo rm -rf "$TOMCAT_SYSLOG_CONFFILE" fi - echo "INFO: Removed all the modified files." - logMsgToConfigSysLog "INFO" "INFO: Tomcat needs to be restarted to rollback the configuration." - restartTomcat + #restart rsyslog + restartRsyslog } #restart tomcat restartTomcat() { - while true; do - read -p "Do you wish to restart tomcat server? (yes/no)" yn - case $yn in - [Yy]* ) - #sudo service tomcat restart or home/bin/start.sh - if [ $(ps -ef | grep -v grep | grep "$SERVICE" | wc -l) -gt 0 ]; then - logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running." - if [ -f /etc/init.d/$SERVICE ]; then - logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running as service." - logMsgToConfigSysLog "INFO" "INFO: Restarting the tomcat service." - sudo service $SERVICE restart - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not restart gracefully. Log rotation may not be disabled. Please restart tomcat manually." - fi - else - logMsgToConfigSysLog "INFO" "INFO: $SERVICE is not running as service." - # To be commented only for test - logMsgToConfigSysLog "INFO" "INFO: Shutting down tomcat." - sudo $LOGGLY_CATALINA_HOME/bin/shutdown.sh - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not shut down gracefully." - else - logMsgToConfigSysLog "INFO" "INFO: Done shutting down tomcat." - fi - - logMsgToConfigSysLog "INFO" "INFO: Starting up tomcat." - sudo $LOGGLY_CATALINA_HOME/bin/startup.sh - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not start up down gracefully." - else - logMsgToConfigSysLog "INFO" "INFO: Tomcat is up and running." - fi - fi + #sudo service tomcat restart or home/bin/start.sh + if [ $(ps -ef | grep -v grep | grep "$SERVICE" | wc -l) -gt 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running." + if [ -f /etc/init.d/$SERVICE ]; then + logMsgToConfigSysLog "INFO" "INFO: $SERVICE is running as service." + logMsgToConfigSysLog "INFO" "INFO: Restarting the tomcat service." + sudo service $SERVICE restart + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not restart gracefully. Log rotation may not be disabled. Please restart tomcat manually." fi - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Exiting the script based on your input. Please restart Tomcat manually." - break;; - * ) echo "Please answer yes or no.";; - esac - done - + else + logMsgToConfigSysLog "INFO" "INFO: $SERVICE is not running as service." + logMsgToConfigSysLog "INFO" "INFO: Shutting down tomcat." + sudo $LOGGLY_CATALINA_HOME/bin/shutdown.sh + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not shut down gracefully." + else + logMsgToConfigSysLog "INFO" "INFO: Done shutting down tomcat." + fi + + logMsgToConfigSysLog "INFO" "INFO: Starting up tomcat." + sudo $LOGGLY_CATALINA_HOME/bin/startup.sh + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: Tomcat did not start up down gracefully." + else + logMsgToConfigSysLog "INFO" "INFO: Tomcat is up and running." + fi + fi + fi } #display usage syntax From dc7792581d26263cb93317958fa33e4331490568 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 4 Jul 2014 21:57:27 +0530 Subject: [PATCH 12/95] No verification if already configured --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 97c11ac..97e8377 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -188,7 +188,7 @@ checkIfSupportedOS() *"Ubuntu"* ) echo "INFO: Operating system is Ubuntu." ;; - *"RedHat"* ) + *"Red Hat"* ) echo "INFO: Operating system is Red Hat." ;; *"CentOS"* ) From e6d46eb0d3c06a0fcaa81a4d771c59380806bc4c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 4 Jul 2014 21:58:44 +0530 Subject: [PATCH 13/95] No verification if already configured --- Linux Script/configure-linux.sh | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 97e8377..0d6b635 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -81,6 +81,9 @@ IS_INVOKED= #this variable will hold if the check env function for linux is invoked LINUX_ENV_VALIDATED= +#this variable will inform if verification needs to be performed +LINUX_DO_VERIFICATION= + ########## Variable Declarations - End ########## #check if the Linux environment is compatible with Loggly. @@ -129,19 +132,20 @@ installLogglyConf() if [ "$LINUX_ENV_VALIDATED" = "" ]; then checkLinuxLogglyCompatibility fi - + #if all the above check passes, write the 22-loggly.conf file write22LogglyConfFile #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu createRsyslogDir - - #check if the logs are going to loggly fro linux system now - checkIfLogsMadeToLoggly - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." - + + if [ "$LINUX_DO_VERIFICATION" = "" ]; then + #check if the logs are going to loggly fro linux system now + checkIfLogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." + fi } #remove loggly configuration from Linux system @@ -188,7 +192,7 @@ checkIfSupportedOS() *"Ubuntu"* ) echo "INFO: Operating system is Ubuntu." ;; - *"Red Hat"* ) + *"RedHat"* ) echo "INFO: Operating system is Red Hat." ;; *"CentOS"* ) @@ -351,15 +355,18 @@ write22LogglyConfFile() if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." while true; do - read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE? (yes/no)" yn + read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn case $yn in [Yy]* ) logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; checkAuthTokenAndWriteContents; break;; - [Nn]* ) break;; - * ) echo "Please answer yes or no.";; + [Nn]* ) + LINUX_DO_VERIFICATION="false" + logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." + break;; + * ) echo "Please answer yes or no.";; esac done else From afb9a36ca6647874296491364756d4c837008812 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 4 Jul 2014 22:00:03 +0530 Subject: [PATCH 14/95] Updated typos and link --- Modular Scripts/Apache2/configure-apache.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 4e9c0e7..df913be 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -36,7 +36,7 @@ LOGGLY_APACHE_LOG_HOME= #this variable will hold the users apache version APACHE_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/apache-application-server" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/" #this variable will hold if the check env function for linux is invoked APACHE_ENV_VALIDATED= @@ -195,6 +195,7 @@ checkLogFileSize() done elif [ $fileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent log files from Apache so verification may not succeed." + exit 1 fi } @@ -330,7 +331,7 @@ usage() { cat << EOF usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-ah apache home (optional)] -usage: configure-linux [-a loggly auth account or subdomain] [-r to rollback] +usage: configure-apache [-a loggly auth account or subdomain] [-r to rollback] usage: configure-apache [-h for help] EOF } @@ -353,7 +354,7 @@ while [ "$1" != "" ]; do ;; -u | --username ) shift LOGGLY_USERNAME=$1 - echo "Userna.loggly.com isme is set" + echo "Username is set" ;; -p | --password ) shift LOGGLY_PASSWORD=$1 From 2868b89e6def1903334f2cbcd43122fe72a16d51 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 9 Jul 2014 09:57:47 +0530 Subject: [PATCH 15/95] Removed duplicate variables and updated message --- Modular Scripts/Apache2/configure-apache.sh | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index df913be..3c642d0 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -20,14 +20,10 @@ SERVICE= APACHE_ACCESS_LOG_FILE= #name of apache error log file APACHE_ERROR_LOG_FILE= -#directory location for syslog -SYSLOG_ETCDIR_CONF=/etc/rsyslog.d #name and location of apache syslog file -APACHE_SYSLOG_CONFFILE=$SYSLOG_ETCDIR_CONF/21-apache.conf +APACHE_SYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/21-apache.conf #name and location of apache syslog backup file -APACHE_SYSLOG_CONFFILE_BACKUP=$SYSLOG_ETCDIR_CONF/21-apache.conf.loggly.bk -#syslog directory -SYSLOG_DIR=/var/spool/rsyslog +APACHE_SYSLOG_CONFFILE_BACKUP=$RSYSLOG_ETCDIR_CONF/21-apache.conf.loggly.bk #this variable will hold the path to the apache home LOGGLY_APACHE_HOME= @@ -231,7 +227,7 @@ write21ApacheFileContents() imfileStr="\$ModLoad imfile \$InputFilePollInterval 10 - \$WorkDirectory $SYSLOG_DIR + \$WorkDirectory $RSYSLOG_DIR " if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then imfileStr+="\$PrivDropToGroup adm @@ -330,7 +326,7 @@ remove21ApacheConfFile() usage() { cat << EOF -usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-ah apache home (optional)] +usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] usage: configure-apache [-a loggly auth account or subdomain] [-r to rollback] usage: configure-apache [-h for help] EOF @@ -371,12 +367,7 @@ while [ "$1" != "" ]; do done fi -if [ "$LOGGLY_DEBUG" != "" -a "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then - if [ "$LOGGLY_PASSWORD" = "" ]; then - getPassword - fi - debug -elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then +if [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi From cb88b8f274bf9e7704bcb1dfb43bd8c46bc8c569 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 9 Jul 2014 09:57:58 +0530 Subject: [PATCH 16/95] Removed duplicate variables --- Modular Scripts/Tomcat/configure-tomcat.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index cdf6e44..775a26a 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -19,14 +19,10 @@ APP_TAG="\"tomcat-version\":\"\"" #name of the service, in this case tomcat6 SERVICE=tomcat6 -#directory location for syslog -SYSLOG_ETCDIR_CONF=/etc/rsyslog.d #name and location of tomcat syslog file -TOMCAT_SYSLOG_CONFFILE=$SYSLOG_ETCDIR_CONF/21-tomcat.conf +TOMCAT_SYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/21-tomcat.conf #name and location of tomcat syslog backup file -TOMCAT_SYSLOG_CONFFILE_BACKUP=$SYSLOG_ETCDIR_CONF/21-tomcat.conf.loggly.bk -#syslog directory -SYSLOG_DIR=/var/spool/rsyslog +TOMCAT_SYSLOG_CONFFILE_BACKUP=$RSYSLOG_ETCDIR_CONF/21-tomcat.conf.loggly.bk #this variable will hold the path to the catalina home LOGGLY_CATALINA_HOME= @@ -344,7 +340,7 @@ canTomcatBeRestarted() [Yy]* ) break;; [Nn]* ) - logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." + logMsgToConfigSysLog "WARN" "WARN: This script must restart Tomcat. Please run the script again when you are ready to restart it. No changes have been made to your system. Exiting." exit 1 break;; * ) echo "Please answer yes or no.";; @@ -449,7 +445,7 @@ write21TomcatFileContents() sudo chmod o+w $TOMCAT_SYSLOG_CONFFILE imfileStr="\$ModLoad imfile - \$WorkDirectory $SYSLOG_DIR + \$WorkDirectory $RSYSLOG_DIR " if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then imfileStr+="\$PrivDropToGroup adm From b0115f96db75dca9fa42cc3eb3c08edd61ab7623 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 9 Jul 2014 17:24:25 +0530 Subject: [PATCH 17/95] Added file monitoring script --- .../configure-file-monitoring.sh | 327 ++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 Modular Scripts/File Monitoring/configure-file-monitoring.sh diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh new file mode 100644 index 0000000..6dd2603 --- /dev/null +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -0,0 +1,327 @@ +#!/bin/bash + +#downloads configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh" +curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +source configure-linux.sh "being-invoked" + +########## Variable Declarations - Start ########## +#name of the current script +SCRIPT_NAME=configure-file-monitoring.sh +#version of the current script +SCRIPT_VERSION=1.0 + +#file to monitor (contains complete path and file name) provided by user +LOGGLY_FILE_TO_MONITOR= + +#alias name, will be used as tag & state file name etc. provided by user +LOGGLY_FILE_TO_MONITOR_ALIAS= + +#file alias provided by the user +APP_TAG="\"file-alias\":\"\"" + +#name and location of syslog file +FILE_SYSLOG_CONFFILE= + +#name and location of syslog backup file +FILE_SYSLOG_CONFFILE_BACKUP= + +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/sending-apache-logs/" + +########## Variable Declarations - End ########## + +# executing the script for loggly to install and configure syslog +installLogglyConfForFile() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating configure Loggly for file monitoring." + + #check if the linux environment is compatible for Loggly + checkLinuxLogglyCompatibility + + #construct variables using filename and filealias + constructVariables + + #check if file to monitor exists + checkIfFileExist + + #check if the alias is already taken + checkIfFileAliasExist + + #configure loggly for Linux + installLogglyConf + + #check for the log file size + checkLogFileSize $LOGGLY_FILE_TO_MONITOR + + #create 21.conf file + write21ConfFileContents + + #verify if the file logs made it to loggly + checkIfFileLogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." +} + +#executing script to remove loggly configuration for File +removeLogglyConfForFile() +{ + logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #construct variables using filename and filealias + constructVariables + + #checks if the conf file exists. if not, then exit. + checkIfConfFileExist + + #remove 21.conf file + remove21ConfFile + + logMsgToConfigSysLog "INFO" "INFO: Rollback completed." +} + +constructVariables() +{ + #conf file name + FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$LOGGLY_FILE_TO_MONITOR_ALIAS.conf" + + #conf file backup name + FILE_SYSLOG_CONFFILE_BACKUP="$FILE_SYSLOG_CONFFILE.loggly.bk" + + #application tag + APP_TAG="\"file-alias\":\"$LOGGLY_FILE_TO_MONITOR_ALIAS\"" +} + +#checks if the file to be monitored exist +checkIfFileExist() +{ + if [ -f "$LOGGLY_FILE_TO_MONITOR" ]; then + logMsgToConfigSysLog "INFO" "INFO: File $LOGGLY_FILE_TO_MONITOR exists." + else + logMsgToConfigSysLog "ERROR" "ERROR: File $LOGGLY_FILE_TO_MONITOR does not exist. Kindly recheck." + exit 1 + fi +} + +#check if the file alias is already taken +checkIfFileAliasExist() +{ + if [ -f "$FILE_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: This file alias is already taken. You must choose a unique file alias for each file." + while true; do + read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + fi +} + +#check the size of the log file. If the size is greater than 100MB give a warning to the user. If the file size is 0 +#then exit +checkLogFileSize() +{ + monitorFileSize=$(wc -c "$1" | cut -f 1 -d ' ') + if [ $monitorFileSize -ge 100000000 ]; then + logMsgToConfigSysLog "INFO" "INFO: " + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size is $LOGGLY_FILE_TO_MONITOR is $monitorFileSize. Continuing with File Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size is $LOGGLY_FILE_TO_MONITOR is $monitorFileSize. Discontinuing with File Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + elif [ $monitorFileSize -eq 0 ]; then + logMsgToConfigSysLog "WARN" "WARN: There are no recent $LOGGLY_FILE_TO_MONITOR log files so verification may not succeed. Exiting." + exit 1 + fi +} + +#function to write the contents of syslog config file +write21ConfFileContents() +{ + logMsgToConfigSysLog "INFO" "INFO: Creating file $FILE_SYSLOG_CONFFILE" + sudo touch $FILE_SYSLOG_CONFFILE + sudo chmod o+w $FILE_SYSLOG_CONFFILE + + imfileStr="\$ModLoad imfile + \$InputFilePollInterval 10 + \$WorkDirectory $RSYSLOG_DIR + " + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + imfileStr+="\$PrivDropToGroup adm + " + fi + + imfileStr+=" + # File access file: + \$InputFileName $LOGGLY_FILE_TO_MONITOR + \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS: + \$InputFileStateFile stat-$LOGGLY_FILE_TO_MONITOR_ALIAS + \$InputFileSeverity info + \$InputFilePersistStateInterval 20000 + \$InputRunFileMonitor + + #Add a tag for file events + \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"file\\\"] %msg%\n\" + + if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;LogglyFormatFile + if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ + " + + #write to 21-.conf file +sudo cat << EOIPFW >> $FILE_SYSLOG_CONFFILE +$imfileStr +EOIPFW + + restartRsyslog +} + +#checks if the apache logs made to loggly +checkIfFileLogsMadeToLoggly() +{ + counter=1 + maxCounter=10 + + fileInitialLogCount=0 + fileLatestLogCount=0 + queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS&from=-15m&until=now&size=1" + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Getting initial log count." + #get the initial count of file logs for past 15 minutes + searchAndFetch fileInitialLogCount "$queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the logs made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + #get the final count of file logs for past 15 minutes + searchAndFetch fileLatestLogCount "$queryUrl" + let counter=$counter+1 + + while [ "$fileLatestLogCount" -le "$fileInitialLogCount" ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch fileLatestLogCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + exit 1 + fi + done + + if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." + exit 0 + fi +} + +#checks if the conf file exist. Name of conf file is constructed using the file alias name provided +checkIfConfFileExist() +{ + if [ ! -f "$FILE_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." + exit 1 + fi +} + +#remove 21.conf file +remove21ConfFile() +{ + echo "INFO: Deleting the loggly file syslog conf file." + if [ -f "$FILE_SYSLOG_CONFFILE" ]; then + sudo rm -rf "$FILE_SYSLOG_CONFFILE" + fi + echo "INFO: Removed all the modified files." + restartRsyslog +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-file [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-f filename] [-l filealias] +usage: configure-file [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] +usage: configure-file [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## + +if [ $# -eq 0 ]; then + usage + exit +else +while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --rollback ) + LOGGLY_ROLLBACK="true" + ;; + -f | --filename ) shift + LOGGLY_FILE_TO_MONITOR=$1 + echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" + ;; + -l | --filealias ) shift + LOGGLY_FILE_TO_MONITOR_ALIAS=$1 + echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" + ;; + -h | --help) + usage + exit + ;; + esac + shift +done +fi + +if [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_FILE_TO_MONITOR" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConfForFile +elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then + removeLogglyConfForFile +else + usage +fi + +########## Get Inputs from User - End ########## \ No newline at end of file From dee3dc33b74c1e0600e52152acd13ce310c8a300 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 10 Jul 2014 23:24:46 +0530 Subject: [PATCH 18/95] Updated Manual link and Logging file name also --- .../configure-file-monitoring.sh | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 6dd2603..cd48261 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -4,7 +4,7 @@ echo "INFO: Downloading dependencies - configure-linux.sh" curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh source configure-linux.sh "being-invoked" - + ########## Variable Declarations - Start ########## #name of the current script SCRIPT_NAME=configure-file-monitoring.sh @@ -26,7 +26,7 @@ FILE_SYSLOG_CONFFILE= #name and location of syslog backup file FILE_SYSLOG_CONFFILE_BACKUP= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/sending-apache-logs/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" ########## Variable Declarations - End ########## @@ -35,28 +35,28 @@ installLogglyConfForFile() { #log message indicating starting of Loggly configuration logMsgToConfigSysLog "INFO" "INFO: Initiating configure Loggly for file monitoring." - + #check if the linux environment is compatible for Loggly checkLinuxLogglyCompatibility - + #construct variables using filename and filealias constructVariables - + #check if file to monitor exists checkIfFileExist - + #check if the alias is already taken checkIfFileAliasExist - + #configure loggly for Linux installLogglyConf #check for the log file size checkLogFileSize $LOGGLY_FILE_TO_MONITOR - + #create 21.conf file write21ConfFileContents - + #verify if the file logs made it to loggly checkIfFileLogsMadeToLoggly @@ -74,10 +74,10 @@ removeLogglyConfForFile() #check if the OS is supported by the script. If no, then exit checkIfSupportedOS - + #construct variables using filename and filealias constructVariables - + #checks if the conf file exists. if not, then exit. checkIfConfFileExist @@ -91,10 +91,10 @@ constructVariables() { #conf file name FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$LOGGLY_FILE_TO_MONITOR_ALIAS.conf" - + #conf file backup name FILE_SYSLOG_CONFFILE_BACKUP="$FILE_SYSLOG_CONFFILE.loggly.bk" - + #application tag APP_TAG="\"file-alias\":\"$LOGGLY_FILE_TO_MONITOR_ALIAS\"" } @@ -122,7 +122,7 @@ checkIfFileAliasExist() logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; break;; - [Nn]* ) + [Nn]* ) logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" exit 1 break;; @@ -137,16 +137,16 @@ checkIfFileAliasExist() checkLogFileSize() { monitorFileSize=$(wc -c "$1" | cut -f 1 -d ' ') - if [ $monitorFileSize -ge 100000000 ]; then + if [ $monitorFileSize -ge 102400000 ]; then logMsgToConfigSysLog "INFO" "INFO: " while true; do read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn case $yn in [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size is $LOGGLY_FILE_TO_MONITOR is $monitorFileSize. Continuing with File Loggly configuration."; + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size is $LOGGLY_FILE_TO_MONITOR is $monitorFileSize. Discontinuing with File Loggly configuration." + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." exit 1 break;; * ) echo "Please answer yes or no.";; @@ -155,7 +155,9 @@ checkLogFileSize() elif [ $monitorFileSize -eq 0 ]; then logMsgToConfigSysLog "WARN" "WARN: There are no recent $LOGGLY_FILE_TO_MONITOR log files so verification may not succeed. Exiting." exit 1 - fi + else + logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." + fi } #function to write the contents of syslog config file @@ -166,7 +168,7 @@ write21ConfFileContents() sudo chmod o+w $FILE_SYSLOG_CONFFILE imfileStr="\$ModLoad imfile - \$InputFilePollInterval 10 + \$InputFilePollInterval 10 \$WorkDirectory $RSYSLOG_DIR " if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then @@ -264,7 +266,7 @@ remove21ConfFile() usage() { cat << EOF -usage: configure-file [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-f filename] [-l filealias] +usage: configure-file [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-l filealias] usage: configure-file [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] usage: configure-file [-h for help] EOF @@ -297,7 +299,8 @@ while [ "$1" != "" ]; do LOGGLY_ROLLBACK="true" ;; -f | --filename ) shift - LOGGLY_FILE_TO_MONITOR=$1 + #LOGGLY_FILE_TO_MONITOR=$1 + LOGGLY_FILE_TO_MONITOR=$(readlink -f $1) echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" ;; -l | --filealias ) shift @@ -313,7 +316,7 @@ while [ "$1" != "" ]; do done fi -if [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_FILE_TO_MONITOR" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then +if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_FILE_TO_MONITOR" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi From 8e21171c4fa1440d31c4933677df2040a7592b50 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 11 Jul 2014 10:46:05 +0530 Subject: [PATCH 19/95] Updated warning text Updated warning text when logs file size is 0 --- Modular Scripts/Apache2/configure-apache.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 3c642d0..c62d5a6 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -180,17 +180,17 @@ checkLogFileSize() read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn case $yn in [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize. Continuing with Apache Loggly configuration."; + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Continuing with Apache Loggly configuration."; break;; [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize. Discontinuing with Apache Loggly configuration." + logMsgToConfigSysLog "INFO" "INFO: Current apache logs size is $fileSize bytes. Discontinuing with Apache Loggly configuration." exit 1 break;; * ) echo "Please answer yes or no.";; esac done elif [ $fileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent log files from Apache so verification may not succeed." + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." exit 1 fi } From 69324a4243a460b8c3c1faa616d7017eabd7f5f5 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 11 Jul 2014 13:54:21 +0530 Subject: [PATCH 20/95] Updated message text Updated message text when there is no data in the configured file. --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index cd48261..bf00ac8 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -153,7 +153,7 @@ checkLogFileSize() esac done elif [ $monitorFileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent $LOGGLY_FILE_TO_MONITOR log files so verification may not succeed. Exiting." + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR there so won't be any data sent to Loggly. You can generate some logs by writing to this file." exit 1 else logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." From 200fd9374fe7063eccaeed1874c84a00d5209383 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 12 Jul 2014 00:03:09 +0530 Subject: [PATCH 21/95] Made token optional --- Linux Script/configure-linux.sh | 44 +++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 0d6b635..088aea0 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -105,7 +105,10 @@ checkLinuxLogglyCompatibility() #check if user credentials are valid. If no, then exit checkIfValidUserNamePassword - #check if authentication token is valid. If no, then exit + #get authentication token if not provided + getAuthToken + + #check if authentication token is valid. If no, then exit. checkIfValidAuthToken #check if rsyslog is configured as service. If no, then exit @@ -132,17 +135,17 @@ installLogglyConf() if [ "$LINUX_ENV_VALIDATED" = "" ]; then checkLinuxLogglyCompatibility fi - + #if all the above check passes, write the 22-loggly.conf file write22LogglyConfFile #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu createRsyslogDir - + if [ "$LINUX_DO_VERIFICATION" = "" ]; then #check if the logs are going to loggly fro linux system now checkIfLogsMadeToLoggly - + #log success message logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." fi @@ -187,7 +190,7 @@ checkIfUserHasRootPrivileges() checkIfSupportedOS() { getOs - + case "$LINUX_DIST" in *"Ubuntu"* ) echo "INFO: Operating system is Ubuntu." @@ -226,7 +229,6 @@ getOs() fi } - #sets linux variables which will be used across various functions setLinuxVariables() { @@ -285,6 +287,28 @@ checkIfValidUserNamePassword() fi } +getAuthToken() +{ + if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." + #get authentication token if user has not provided one + tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") + + #get the string from index 0 to first occurence of , + tokenstr=${tokenstr%%,*} + + #get the string from index 0 to last occurence of " + tokenstr=${tokenstr%\"*} + + #get the string from first occurence of " to the end + tokenstr=${tokenstr#*\"} + + LOGGLY_AUTH_TOKEN=$tokenstr + + logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" + fi +} + #check if authentication token is valid checkIfValidAuthToken() { @@ -362,11 +386,11 @@ write22LogglyConfFile() sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; checkAuthTokenAndWriteContents; break;; - [Nn]* ) + [Nn]* ) LINUX_DO_VERIFICATION="false" logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." break;; - * ) echo "Please answer yes or no.";; + * ) echo "Please answer yes or no.";; esac done else @@ -605,7 +629,7 @@ getPassword() usage() { cat << EOF -usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] +usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] usage: configure-linux [-a loggly auth account or subdomain] [-r to remove] usage: configure-linux [-h for help] EOF @@ -651,7 +675,7 @@ if [ "$1" != "being-invoked" ]; then if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then removeLogglyConf - elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi From 4bfbbc03a13b9b62a6ba0f4fd2d739599c9b5f5c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 12 Jul 2014 00:04:43 +0530 Subject: [PATCH 22/95] Made token optional --- Modular Scripts/Apache2/configure-apache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index c62d5a6..d84314f 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -326,7 +326,7 @@ remove21ApacheConfFile() usage() { cat << EOF -usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] +usage: configure-apache [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] usage: configure-apache [-a loggly auth account or subdomain] [-r to rollback] usage: configure-apache [-h for help] EOF @@ -367,7 +367,7 @@ while [ "$1" != "" ]; do done fi -if [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then +if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi From 6f4412f111283a14e3c7e2fa966383ef804e2886 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 12 Jul 2014 00:05:55 +0530 Subject: [PATCH 23/95] Made token optional --- Modular Scripts/Tomcat/configure-tomcat.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 775a26a..b3d11ea 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -643,7 +643,7 @@ restartTomcat() usage() { cat << EOF -usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token] [-u username] [-p password (optional)] [-ch catalina home (optional)] +usage: configure-tomcat [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-ch catalina home (optional)] usage: configure-tomcat [-r to rollback] [-a loggly auth account or subdomain] [-ch catalina home (optional)] usage: configure-tomcat [-h for help] EOF @@ -688,12 +688,12 @@ while [ "$1" != "" ]; do done fi -if [ "$LOGGLY_DEBUG" != "" -a "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then +if [ "$LOGGLY_DEBUG" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi debug -elif [ "$LOGGLY_AUTH_TOKEN" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then +elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi From a1b6222ed12d34ec6481f8e351e5f40c9739bbe3 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 14 Jul 2014 23:11:44 +0530 Subject: [PATCH 24/95] Check if the file name/ path contains spaces --- .../configure-file-monitoring.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index bf00ac8..6426a15 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -39,6 +39,9 @@ installLogglyConfForFile() #check if the linux environment is compatible for Loggly checkLinuxLogglyCompatibility + #checks if the file name contain spaces, if yes, the exit + checkIfFileLocationContainSpaces + #construct variables using filename and filealias constructVariables @@ -87,6 +90,16 @@ removeLogglyConfForFile() logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } +checkIfFileLocationContainSpaces() +{ + case "$LOGGLY_FILE_TO_MONITOR" in + *\ * ) + logMsgToConfigSysLog "ERROR" "ERROR: File location cannot contain spaces." + exit 1;; + *) break;; + esac +} + constructVariables() { #conf file name @@ -153,7 +166,7 @@ checkLogFileSize() esac done elif [ $monitorFileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR there so won't be any data sent to Loggly. You can generate some logs by writing to this file." + logMsgToConfigSysLog "WARN" "WARN: There are no recent $LOGGLY_FILE_TO_MONITOR log files so verification may not succeed. Exiting." exit 1 else logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." @@ -178,7 +191,7 @@ write21ConfFileContents() imfileStr+=" # File access file: - \$InputFileName $LOGGLY_FILE_TO_MONITOR + \$InputFileName \"$LOGGLY_FILE_TO_MONITOR\" \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS: \$InputFileStateFile stat-$LOGGLY_FILE_TO_MONITOR_ALIAS \$InputFileSeverity info @@ -300,7 +313,7 @@ while [ "$1" != "" ]; do ;; -f | --filename ) shift #LOGGLY_FILE_TO_MONITOR=$1 - LOGGLY_FILE_TO_MONITOR=$(readlink -f $1) + LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" ;; -l | --filealias ) shift From 1ae7d6b7274434be97c4926da3dd18ea2b82ace4 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 14 Jul 2014 23:28:12 +0530 Subject: [PATCH 25/95] Removed Extra characters --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 6426a15..9f9cf99 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -191,7 +191,7 @@ write21ConfFileContents() imfileStr+=" # File access file: - \$InputFileName \"$LOGGLY_FILE_TO_MONITOR\" + \$InputFileName $LOGGLY_FILE_TO_MONITOR \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS: \$InputFileStateFile stat-$LOGGLY_FILE_TO_MONITOR_ALIAS \$InputFileSeverity info From fe0a452aec3e233e3abf20f07038457ec674262c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 14 Jul 2014 23:45:33 +0530 Subject: [PATCH 26/95] Updated version 1.0 to 1.01 --- Modular Scripts/Tomcat/configure-tomcat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index b3d11ea..bc92fbe 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-tomcat.sh #version of the current script -SCRIPT_VERSION=1.0 +SCRIPT_VERSION=1.01 #minimum version of tomcat to enable log rotation MIN_TOMCAT_VERSION=6.0.33.0 From 0cf4b30337789329e8e0ea15e3dfc3c2a750f1e6 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 15 Jul 2014 11:46:59 +0530 Subject: [PATCH 27/95] Updated message --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 9f9cf99..ed03fab 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -166,7 +166,7 @@ checkLogFileSize() esac done elif [ $monitorFileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent $LOGGLY_FILE_TO_MONITOR log files so verification may not succeed. Exiting." + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR there so won't be any data sent to Loggly. You can generate some logs by writing to this file." exit 1 else logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." From 1d22a455cdc2b76d4f4d76787812fe7c80b6495a Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 15 Jul 2014 17:54:11 +0530 Subject: [PATCH 28/95] Removed extra break --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index ed03fab..361d41b 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -96,7 +96,7 @@ checkIfFileLocationContainSpaces() *\ * ) logMsgToConfigSysLog "ERROR" "ERROR: File location cannot contain spaces." exit 1;; - *) break;; + *) ;; esac } @@ -166,7 +166,7 @@ checkLogFileSize() esac done elif [ $monitorFileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR there so won't be any data sent to Loggly. You can generate some logs by writing to this file." + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR so there won't be any data sent to Loggly. You can generate some logs by writing to this file." exit 1 else logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." From d63e96db043d9e3fb439a04462636830c5034e7c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 15 Jul 2014 17:56:31 +0530 Subject: [PATCH 29/95] Updated text message --- Modular Scripts/Apache2/configure-apache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index d84314f..f2839fa 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -190,7 +190,7 @@ checkLogFileSize() esac done elif [ $fileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache so there won't be any data sent to Loggly. You can generate some logs by visiting a page on your web server." exit 1 fi } From c6197dfc520111d9c90a0005c00306cac71d8fd8 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 16 Jul 2014 18:49:32 +0530 Subject: [PATCH 30/95] Configure script if log file size is 0 --- Modular Scripts/Apache2/configure-apache.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index f2839fa..23d5260 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-apache.sh #version of the current script -SCRIPT_VERSION=1.0 +SCRIPT_VERSION=1.01 #we have not found the apache version yet at this point in the script APP_TAG="\"apache-version\":\"\"" @@ -65,12 +65,12 @@ installLogglyConfForApache() #configure loggly for Linux installLogglyConf - #check for the apache log file size - checkLogFileSize $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE - #create 21apache.conf file write21ApacheConfFile + #check for the apache log file size + checkLogFileSize $LOGGLY_APACHE_LOG_HOME/$APACHE_ACCESS_LOG_FILE $LOGGLY_APACHE_LOG_HOME/$APACHE_ERROR_LOG_FILE + #verify if the apache logs made it to loggly checkIfApacheLogsMadeToLoggly @@ -190,7 +190,7 @@ checkLogFileSize() esac done elif [ $fileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache so there won't be any data sent to Loggly. You can generate some logs by visiting a page on your web server." + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from Apache there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." exit 1 fi } From 9a98ed3ded5171cfa746f29d14041e6a260f0324 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 16 Jul 2014 18:50:16 +0530 Subject: [PATCH 31/95] Configure Script if log file size is 0 --- .../configure-file-monitoring.sh | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 361d41b..b0e234c 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.0 +SCRIPT_VERSION=1.01 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -43,7 +43,7 @@ installLogglyConfForFile() checkIfFileLocationContainSpaces #construct variables using filename and filealias - constructVariables + constructFileVariables #check if file to monitor exists checkIfFileExist @@ -54,12 +54,15 @@ installLogglyConfForFile() #configure loggly for Linux installLogglyConf - #check for the log file size - checkLogFileSize $LOGGLY_FILE_TO_MONITOR - #create 21.conf file write21ConfFileContents + #restart rsyslog + restartRsyslog + + #check for the log file size + checkLogFileSize $LOGGLY_FILE_TO_MONITOR + #verify if the file logs made it to loggly checkIfFileLogsMadeToLoggly @@ -79,14 +82,18 @@ removeLogglyConfForFile() checkIfSupportedOS #construct variables using filename and filealias - constructVariables + constructFileVariables #checks if the conf file exists. if not, then exit. checkIfConfFileExist #remove 21.conf file remove21ConfFile - + + #restart rsyslog + restartRsyslog + + #log success message logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } @@ -100,7 +107,7 @@ checkIfFileLocationContainSpaces() esac } -constructVariables() +constructFileVariables() { #conf file name FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$LOGGLY_FILE_TO_MONITOR_ALIAS.conf" @@ -210,7 +217,6 @@ sudo cat << EOIPFW >> $FILE_SYSLOG_CONFFILE $imfileStr EOIPFW - restartRsyslog } #checks if the apache logs made to loggly @@ -272,16 +278,15 @@ remove21ConfFile() sudo rm -rf "$FILE_SYSLOG_CONFFILE" fi echo "INFO: Removed all the modified files." - restartRsyslog } #display usage syntax usage() { cat << EOF -usage: configure-file [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-l filealias] -usage: configure-file [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] -usage: configure-file [-h for help] +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-l filealias] +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] +usage: configure-file-monitoring [-h for help] EOF } From 8bffdcada1a555d11d4153a84144c50e7292b575 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 18 Jul 2014 17:28:24 +0530 Subject: [PATCH 32/95] Added README.md for the Linux Script --- Linux Script/README.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Linux Script/README.md diff --git a/Linux Script/README.md b/Linux Script/README.md new file mode 100644 index 0000000..256dae0 --- /dev/null +++ b/Linux Script/README.md @@ -0,0 +1,2 @@ +Linux Script +============ From 35323ad88021edcea9ad6ce4f5dc8c1376aba23b Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 18 Jul 2014 17:35:19 +0530 Subject: [PATCH 33/95] Updated README --- Linux Script/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Linux Script/README.md b/Linux Script/README.md index 256dae0..68b91fd 100644 --- a/Linux Script/README.md +++ b/Linux Script/README.md @@ -1,2 +1,12 @@ Linux Script ============ + +Configure your Linux system to send syslogs to Loggly using the following command + + chmod 755 configure-linux.sh + sudo ./configure-linux.sh -a SUBDOMAIN -u USERNAME + + +Stop sending your Linux System logs to Loggly + + sudo ./configure-linux.sh -a SUBDOMAIN -r From 5d6e636451e379b6ff9b182ae202fc7b55b5eb22 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 18 Jul 2014 17:40:39 +0530 Subject: [PATCH 34/95] Create README.md --- Modular Scripts/Apache2/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Modular Scripts/Apache2/README.md diff --git a/Modular Scripts/Apache2/README.md b/Modular Scripts/Apache2/README.md new file mode 100644 index 0000000..621f31e --- /dev/null +++ b/Modular Scripts/Apache2/README.md @@ -0,0 +1,11 @@ +Apache Script +============= + +Configure your Apache server to send logs from access file and error file to Loggly + + chmod 755 configure-apache.sh + sudo ./configure-apache -a SUBDOMAIN -u USERNAME + +Stop sending your Apache logs to Loggly + + sudo ./configure-apache.sh -a SUBDOMAIN -r From 3752d2cf0e53c8776df462cccbf047feed02b02f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 18 Jul 2014 17:44:53 +0530 Subject: [PATCH 35/95] Create README.md --- Modular Scripts/File Monitoring/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 Modular Scripts/File Monitoring/README.md diff --git a/Modular Scripts/File Monitoring/README.md b/Modular Scripts/File Monitoring/README.md new file mode 100644 index 0000000..b438df5 --- /dev/null +++ b/Modular Scripts/File Monitoring/README.md @@ -0,0 +1,13 @@ +File Monitoring Script +====================== + +Configure your any text file to send it contents to Loggly + + chmod 755 configure-file-monitoring.sh + sudo ./configure-file-monitoring.sh -a SUBDOMAIN -u USERNAME -f FILENAME -l FILE_ALIAS + + Note: File Alias should be unique for each file. + +Stop sending your file contents to Loggly + + sudo ./configure-file-monitoring.sh -a SUBDOMAIN -l FILE_ALIAS -r From c4bf0940d60f0506f3dcbb26cf22d004f8769745 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 18 Jul 2014 17:46:33 +0530 Subject: [PATCH 36/95] Update README.md --- Modular Scripts/File Monitoring/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/README.md b/Modular Scripts/File Monitoring/README.md index b438df5..8bc5bd5 100644 --- a/Modular Scripts/File Monitoring/README.md +++ b/Modular Scripts/File Monitoring/README.md @@ -6,7 +6,7 @@ Configure your any text file to send it contents to Loggly chmod 755 configure-file-monitoring.sh sudo ./configure-file-monitoring.sh -a SUBDOMAIN -u USERNAME -f FILENAME -l FILE_ALIAS - Note: File Alias should be unique for each file. +**Note:** File Alias should be unique for each file. Stop sending your file contents to Loggly From 04891f1435bac8e9f6e49bd294c537b7f1c123db Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 18 Jul 2014 17:46:49 +0530 Subject: [PATCH 37/95] Update README.md --- Modular Scripts/File Monitoring/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modular Scripts/File Monitoring/README.md b/Modular Scripts/File Monitoring/README.md index 8bc5bd5..21d1797 100644 --- a/Modular Scripts/File Monitoring/README.md +++ b/Modular Scripts/File Monitoring/README.md @@ -8,6 +8,8 @@ Configure your any text file to send it contents to Loggly **Note:** File Alias should be unique for each file. + + Stop sending your file contents to Loggly sudo ./configure-file-monitoring.sh -a SUBDOMAIN -l FILE_ALIAS -r From 14f566df0ed115fda96288a944ed7977f4a202e9 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 18 Jul 2014 17:49:30 +0530 Subject: [PATCH 38/95] Create README.md --- Modular Scripts/Tomcat/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Modular Scripts/Tomcat/README.md diff --git a/Modular Scripts/Tomcat/README.md b/Modular Scripts/Tomcat/README.md new file mode 100644 index 0000000..e06c5e7 --- /dev/null +++ b/Modular Scripts/Tomcat/README.md @@ -0,0 +1,11 @@ +Tomcat Script +============= + +Send your Tomcat logs to Loggly + + chmod 755 configure-tomcat.sh + sudo ./configure-tomcat.sh -a SUBDOMAIN -u USERNAME + +Stop sending your Tomcat logs to Loggly + + sudo ./configure-tomcat.sh -a SUBDOMAIN -r From 138ef2f2e96403ba8c87c43130a6036be81b996a Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 21 Jul 2014 20:52:11 +0530 Subject: [PATCH 39/95] Incorporated Manoj's Feedback --- Modular Scripts/Apache2/configure-apache.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 23d5260..d4cb448 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-apache.sh #version of the current script -SCRIPT_VERSION=1.01 +SCRIPT_VERSION=1.1 #we have not found the apache version yet at this point in the script APP_TAG="\"apache-version\":\"\"" @@ -35,7 +35,7 @@ APACHE_VERSION= MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure Apache2 is available at https://www.loggly.com/docs/sending-apache-logs/" #this variable will hold if the check env function for linux is invoked -APACHE_ENV_VALIDATED= +APACHE_ENV_VALIDATED="false" ########## Variable Declarations - End ########## #check if apache environment is compatible for Loggly @@ -58,7 +58,7 @@ installLogglyConfForApache() logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Apache." #check if apache environment is compatible with Loggly - if [ "$APACHE_ENV_VALIDATED" = "" ]; then + if [ "$APACHE_ENV_VALIDATED" = "false" ]; then checkApacheLogglyCompatibility fi @@ -155,7 +155,7 @@ getApacheVersion() APACHE_VERSION=${APACHE_VERSION% *} APACHE_VERSION=$APACHE_VERSION | tr -d ' ' APP_TAG="\"apache-version\":\"$APACHE_VERSION\"" - echo "Apache version: " $APACHE_VERSION + logMsgToConfigSysLog "INFO" "INFO: Apache version: $APACHE_VERSION" } #checks if the apache version is supported by this script, currently the script @@ -174,7 +174,7 @@ checkLogFileSize() accessFileSize=$(wc -c "$1" | cut -f 1 -d ' ') errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') fileSize=$((accessFileSize+errorFileSize)) - if [ $fileSize -ge 100000000 ]; then + if [ $fileSize -ge 102400000 ]; then logMsgToConfigSysLog "INFO" "INFO: " while true; do read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn From aac29fa0a6572f53dc0811827c3cbad9cb0456d9 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 21 Jul 2014 21:19:44 +0530 Subject: [PATCH 40/95] Incorporated Manoj's Feedback --- Linux Script/configure-linux.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 088aea0..26b2248 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.01 +SCRIPT_VERSION=1.1 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -79,10 +79,10 @@ MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are IS_INVOKED= #this variable will hold if the check env function for linux is invoked -LINUX_ENV_VALIDATED= +LINUX_ENV_VALIDATED="false" #this variable will inform if verification needs to be performed -LINUX_DO_VERIFICATION= +LINUX_DO_VERIFICATION="false" ########## Variable Declarations - End ########## @@ -132,7 +132,7 @@ installLogglyConf() #log message indicating starting of Loggly configuration logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." - if [ "$LINUX_ENV_VALIDATED" = "" ]; then + if [ "$LINUX_ENV_VALIDATED" = "false" ]; then checkLinuxLogglyCompatibility fi @@ -142,7 +142,7 @@ installLogglyConf() #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu createRsyslogDir - if [ "$LINUX_DO_VERIFICATION" = "" ]; then + if [ "$LINUX_DO_VERIFICATION" = "true" ]; then #check if the logs are going to loggly fro linux system now checkIfLogsMadeToLoggly @@ -382,6 +382,7 @@ write22LogglyConfFile() read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn case $yn in [Yy]* ) + LINUX_DO_VERIFICATION="true" logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; checkAuthTokenAndWriteContents; From b45d7e9902d6168a7fafe25f3b946df9ee2da15d Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 21 Jul 2014 21:41:04 +0530 Subject: [PATCH 41/95] Update README.md --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4a9b276..51af319 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ install-script ============== -Loggly Sender Installation Script. +Loggly Sender Installation Script. Sends Linux, Tomcat, Apache and any file logs to Loggly. -Versions are tagged with the date they are pushed to the S3 bucket. Tag format is: +***Note:*** These scripts supports only Loggly **Gen2** account. -yyyy-mm-dd-n -where n is a number, increasing if the same date is re-used. +You can view the **READMEs** of the various installation scripts at the following paths + + +1. Linux Configuration Script +2. Apache Installation Script +3. File Monitoring Configuration Script +4. Tomcat Configuration Script From be5085dc7d26cbad3097caecb471ec76829a7f51 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 21 Jul 2014 21:41:43 +0530 Subject: [PATCH 42/95] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 51af319..ca9d31c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ install-script ============== -Loggly Sender Installation Script. Sends Linux, Tomcat, Apache and any file logs to Loggly. +Loggly Sender Installation Script. Sends Linux, Tomcat, Apache and Text file logs to Loggly. ***Note:*** These scripts supports only Loggly **Gen2** account. From 2364a89e83684f36dc5922a3e44d2c44d8a24fba Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 21 Jul 2014 21:45:45 +0530 Subject: [PATCH 43/95] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca9d31c..78b4585 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ install-script Loggly Sender Installation Script. Sends Linux, Tomcat, Apache and Text file logs to Loggly. -***Note:*** These scripts supports only Loggly **Gen2** account. +***Note:*** These scripts support only Loggly **Gen2** account. You can view the **READMEs** of the various installation scripts at the following paths From e96a030710e71219e25741b3e48f44638f00f350 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 21 Jul 2014 22:13:10 +0530 Subject: [PATCH 44/95] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78b4585..96b84fa 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ install-script ============== -Loggly Sender Installation Script. Sends Linux, Tomcat, Apache and Text file logs to Loggly. +Loggly Sender Installation Script. Sends Linux, Tomcat, Apache and Text file logs/ contents to Loggly. ***Note:*** These scripts support only Loggly **Gen2** account. From 4c2ffe607352e99602344c7bbe3b4a111154c08f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 24 Jul 2014 00:06:25 +0530 Subject: [PATCH 45/95] Updated script to support any Linux OS --- Linux Script/configure-linux.sh | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 26b2248..83d5ad5 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -82,7 +82,7 @@ IS_INVOKED= LINUX_ENV_VALIDATED="false" #this variable will inform if verification needs to be performed -LINUX_DO_VERIFICATION="false" +LINUX_DO_VERIFICATION="true" ########## Variable Declarations - End ########## @@ -190,21 +190,38 @@ checkIfUserHasRootPrivileges() checkIfSupportedOS() { getOs - - case "$LINUX_DIST" in - *"Ubuntu"* ) + + LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") + + case "$LINUX_DIST_IN_LOWER_CASE" in + *"ubuntu"* ) echo "INFO: Operating system is Ubuntu." ;; - *"RedHat"* ) + *"redhat"* ) echo "INFO: Operating system is Red Hat." ;; - *"CentOS"* ) + *"centos"* ) echo "INFO: Operating system is CentOS." ;; - * ) + *"darwin"* ) + #if the OS is mac then exit logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." exit 1 ;; + * ) + logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." + while true; do + read -p "Would you like to continue anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + exit 1 + ;; + * ) echo "Please answer yes or no.";; + esac + done + ;; esac } @@ -382,7 +399,6 @@ write22LogglyConfFile() read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn case $yn in [Yy]* ) - LINUX_DO_VERIFICATION="true" logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; checkAuthTokenAndWriteContents; From 85df4509acb3c02e88b433d3f21d2ddc13ff8a11 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sat, 26 Jul 2014 00:04:08 +0530 Subject: [PATCH 46/95] Added s3 monitoring scripts Under development --- .../configure-file-monitoring.sh | 357 +++++++++ .../S3Logs Monitoring/configure-linux.sh | 712 ++++++++++++++++++ .../configure-s3-file-monitoring.sh | 419 +++++++++++ 3 files changed, 1488 insertions(+) create mode 100644 Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh create mode 100644 Modular Scripts/S3Logs Monitoring/configure-linux.sh create mode 100644 Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh diff --git a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh new file mode 100644 index 0000000..5482aaf --- /dev/null +++ b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh @@ -0,0 +1,357 @@ +#!/bin/bash + +#downloads configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh" +#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +source configure-linux.sh "being-invoked" + +########## Variable Declarations - Start ########## +#name of the current script +SCRIPT_NAME=configure-file-monitoring.sh +#version of the current script +SCRIPT_VERSION=1.1 + +#file to monitor (contains complete path and file name) provided by user +LOGGLY_FILE_TO_MONITOR= + +#alias name, will be used as tag & state file name etc. provided by user +LOGGLY_FILE_TO_MONITOR_ALIAS= + +#file alias provided by the user +APP_TAG="\"file-alias\":\"\"" + +#name and location of syslog file +FILE_SYSLOG_CONFFILE= + +#name and location of syslog backup file +FILE_SYSLOG_CONFFILE_BACKUP= + +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" + +#this variable is set if the script is invoked via some other calling script +IS_FILE_MONITOR_SCRIPT_INVOKED="false" + +########## Variable Declarations - End ########## + +# executing the script for loggly to install and configure syslog +installLogglyConfForFile() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating configure Loggly for file monitoring." + + #check if the linux environment is compatible for Loggly + checkLinuxLogglyCompatibility + + #checks if the file name contain spaces, if yes, the exit + checkIfFileLocationContainSpaces + + #construct variables using filename and filealias + constructFileVariables + + #check if file to monitor exists + checkIfFileExist + + #check if the alias is already taken + checkIfFileAliasExist + + #configure loggly for Linux + installLogglyConf + + #create 21.conf file + write21ConfFileContents + + #restart rsyslog + restartRsyslog + + #check for the log file size + checkLogFileSize $LOGGLY_FILE_TO_MONITOR + + #verify if the file logs made it to loggly + checkIfFileLogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." +} + +#executing script to remove loggly configuration for File +removeLogglyConfForFile() +{ + logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #construct variables using filename and filealias + constructFileVariables + + #checks if the conf file exists. if not, then exit. + checkIfConfFileExist + + #remove 21.conf file + remove21ConfFile + + #restart rsyslog + restartRsyslog + + #log success message + logMsgToConfigSysLog "INFO" "INFO: Rollback completed." +} + +checkIfFileLocationContainSpaces() +{ + case "$LOGGLY_FILE_TO_MONITOR" in + *\ * ) + logMsgToConfigSysLog "ERROR" "ERROR: File location cannot contain spaces." + exit 1;; + *) ;; + esac +} + +constructFileVariables() +{ + #conf file name + FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$LOGGLY_FILE_TO_MONITOR_ALIAS.conf" + + #conf file backup name + FILE_SYSLOG_CONFFILE_BACKUP="$FILE_SYSLOG_CONFFILE.loggly.bk" + + #application tag + APP_TAG="\"file-alias\":\"$LOGGLY_FILE_TO_MONITOR_ALIAS\"" +} + +#checks if the file to be monitored exist +checkIfFileExist() +{ + if [ -f "$LOGGLY_FILE_TO_MONITOR" ]; then + logMsgToConfigSysLog "INFO" "INFO: File $LOGGLY_FILE_TO_MONITOR exists." + else + logMsgToConfigSysLog "ERROR" "ERROR: File $LOGGLY_FILE_TO_MONITOR does not exist. Kindly recheck." + exit 1 + fi +} + +#check if the file alias is already taken +checkIfFileAliasExist() +{ + if [ -f "$FILE_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: This file alias is already taken. You must choose a unique file alias for each file." + while true; do + read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + fi +} + +#check the size of the log file. If the size is greater than 100MB give a warning to the user. If the file size is 0 +#then exit +checkLogFileSize() +{ + monitorFileSize=$(wc -c "$1" | cut -f 1 -d ' ') + if [ $monitorFileSize -ge 102400000 ]; then + logMsgToConfigSysLog "INFO" "INFO: " + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + elif [ $monitorFileSize -eq 0 ]; then + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR so there won't be any data sent to Loggly. You can generate some logs by writing to this file." + exit 1 + else + logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." + fi +} + +#function to write the contents of syslog config file +write21ConfFileContents() +{ + logMsgToConfigSysLog "INFO" "INFO: Creating file $FILE_SYSLOG_CONFFILE" + sudo touch $FILE_SYSLOG_CONFFILE + sudo chmod o+w $FILE_SYSLOG_CONFFILE + + imfileStr="\$ModLoad imfile + \$InputFilePollInterval 10 + \$WorkDirectory $RSYSLOG_DIR + " + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + imfileStr+="\$PrivDropToGroup adm + " + fi + + imfileStr+=" + # File access file: + \$InputFileName $LOGGLY_FILE_TO_MONITOR + \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS: + \$InputFileStateFile stat-$LOGGLY_FILE_TO_MONITOR_ALIAS + \$InputFileSeverity info + \$InputFilePersistStateInterval 20000 + \$InputRunFileMonitor + + #Add a tag for file events + \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"file\\\"] %msg%\n\" + + if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;LogglyFormatFile + if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ + " + + #write to 21-.conf file +sudo cat << EOIPFW >> $FILE_SYSLOG_CONFFILE +$imfileStr +EOIPFW + +} + +#checks if the apache logs made to loggly +checkIfFileLogsMadeToLoggly() +{ + counter=1 + maxCounter=10 + + fileInitialLogCount=0 + fileLatestLogCount=0 + queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS&from=-15m&until=now&size=1" + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Getting initial log count." + #get the initial count of file logs for past 15 minutes + searchAndFetch fileInitialLogCount "$queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the logs made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + #get the final count of file logs for past 15 minutes + searchAndFetch fileLatestLogCount "$queryUrl" + let counter=$counter+1 + + while [ "$fileLatestLogCount" -le "$fileInitialLogCount" ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch fileLatestLogCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + exit 1 + fi + done + + if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then + exit 0 + fi + fi +} + +#checks if the conf file exist. Name of conf file is constructed using the file alias name provided +checkIfConfFileExist() +{ + if [ ! -f "$FILE_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." + exit 1 + fi +} + +#remove 21.conf file +remove21ConfFile() +{ + echo "INFO: Deleting the loggly file syslog conf file." + if [ -f "$FILE_SYSLOG_CONFFILE" ]; then + sudo rm -rf "$FILE_SYSLOG_CONFFILE" + echo "INFO: Removed all the modified files." + else + logMsgToConfigSysLog "WARN" "WARN: $FILE_SYSLOG_CONFFILE file was not found." + fi +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-l filealias] +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] +usage: configure-file-monitoring [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## +if [ "$1" != "being-invoked" ]; then + if [ $# -eq 0 ]; then + usage + exit + else + while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --rollback ) + LOGGLY_ROLLBACK="true" + ;; + -f | --filename ) shift + #LOGGLY_FILE_TO_MONITOR=$1 + LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") + echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" + ;; + -l | --filealias ) shift + LOGGLY_FILE_TO_MONITOR_ALIAS=$1 + echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" + ;; + -h | --help) + usage + exit + ;; + esac + shift + done + fi + + if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_FILE_TO_MONITOR" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConfForFile + elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then + removeLogglyConfForFile + else + usage + fi +else + IS_FILE_MONITOR_SCRIPT_INVOKED="true" +fi +########## Get Inputs from User - End ########## \ No newline at end of file diff --git a/Modular Scripts/S3Logs Monitoring/configure-linux.sh b/Modular Scripts/S3Logs Monitoring/configure-linux.sh new file mode 100644 index 0000000..234a623 --- /dev/null +++ b/Modular Scripts/S3Logs Monitoring/configure-linux.sh @@ -0,0 +1,712 @@ +#!/bin/bash + +#trapping Control + C +#these statements must be the first statements in the script to trap the CTRL C event + +trap ctrl_c INT + +function ctrl_c() { + logMsgToConfigSysLog "INFO" "INFO: Aborting the script." + exit 1 +} + +########## Variable Declarations - Start ########## + +#name of the current script. This will get overwritten by the child script which calls this +SCRIPT_NAME=configure-linux.sh +#version of the current script. This will get overwritten by the child script which calls this +SCRIPT_VERSION=1.1 + +#application tag. This will get overwritten by the child script which calls this +APP_TAG= + +#directory location for syslog +RSYSLOG_ETCDIR_CONF=/etc/rsyslog.d +#name and location of loggly syslog file +LOGGLY_RSYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/22-loggly.conf +#name and location of loggly syslog backup file +LOGGLY_RSYSLOG_CONFFILE_BACKUP=$LOGGLY_RSYSLOG_CONFFILE.loggly.bk + +#syslog directory +RSYSLOG_DIR=/var/spool/rsyslog +#rsyslog service name +RSYSLOG_SERVICE=rsyslog +#rsyslogd +RSYSLOGD=rsyslogd +#minimum version of rsyslog to enable logging to loggly +MIN_RSYSLOG_VERSION=5.8.0 +#this variable will hold the users syslog version +RSYSLOG_VERSION= + +#this variable will hold the host name +HOST_NAME= +#this variable will hold the name of the linux distribution +LINUX_DIST= + +#host name for logs-01.loggly.com +LOGS_01_HOST=logs-01.loggly.com +LOGS_01_URL=https://$LOGS_01_HOST +#this variable will contain loggly account url in the format https://$LOGGLY_ACCOUNT.loggly.com +LOGGLY_ACCOUNT_URL= +#loggly.com URL +LOGGLY_COM_URL=https://www.loggly.com + +######Inputs provided by user###### +#this variable will hold the loggly account name provided by user. +#this is a mandatory input +LOGGLY_ACCOUNT= +#this variable will hold the loggly authentication token provided by user. +#this is a mandatory input +LOGGLY_AUTH_TOKEN= +#this variable will identify if the user has selected to rollback settings +LOGGLY_ROLLBACK= +#this variable will hold the user name provided by user +#this is a mandatory input +LOGGLY_USERNAME= +#this variable will hold the password provided by user +#this is a mandatory input +LOGGLY_PASSWORD= + +#variables used in 22-loggly.conf file +LOGGLY_SYSLOG_PORT=514 +LOGGLY_DISTRIBUTION_ID="41058" + +#Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this +#on how to configure the child application +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." + +#this variable is set if the script is invoked via some other calling script +IS_INVOKED= + +#this variable will hold if the check env function for linux is invoked +LINUX_ENV_VALIDATED="false" + +#this variable will inform if verification needs to be performed +LINUX_DO_VERIFICATION="true" + +########## Variable Declarations - End ########## + +#check if the Linux environment is compatible with Loggly. +#Also set few variables after the check. +checkLinuxLogglyCompatibility() +{ + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setLinuxVariables + + #check if the Loggly servers are accessible. If no, ask user to check network connectivity & exit + checkIfLogglyServersAccessible + + #check if user credentials are valid. If no, then exit + checkIfValidUserNamePassword + + #get authentication token if not provided + getAuthToken + + #check if authentication token is valid. If no, then exit. + checkIfValidAuthToken + + #check if rsyslog is configured as service. If no, then exit + checkIfRsyslogConfiguredAsService + + #check if multiple rsyslog are present in the system. If yes, then exit + checkIfMultipleRsyslogConfigured + + #check for the minimum version of rsyslog i.e 5.8.0. If no, then exit + checkIfMinVersionOfRsyslog + + #check if selinux service is enforced. if yes, ask the user to manually disable and exit the script + checkIfSelinuxServiceEnforced + + LINUX_ENV_VALIDATED="true" +} + +# executing the script for loggly to install and configure rsyslog. +installLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." + + if [ "$LINUX_ENV_VALIDATED" = "false" ]; then + checkLinuxLogglyCompatibility + fi + + #if all the above check passes, write the 22-loggly.conf file + write22LogglyConfFile + + #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu + createRsyslogDir + + if [ "$LINUX_DO_VERIFICATION" = "true" ]; then + #check if the logs are going to loggly fro linux system now + checkIfLogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." + fi +} + +#remove loggly configuration from Linux system +removeLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating uninstall Loggly for Linux." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setLinuxVariables + + #remove 22-loggly.conf file + remove22LogglyConfFile + + #restart rsyslog service + restartRsyslog + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Uninstalled Loggly configuration from Linux system." +} + +#checks if user has root privileges +checkIfUserHasRootPrivileges() +{ + #This script needs to be run as a sudo user + if [[ $EUID -ne 0 ]]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script must be run as root." + exit 1 + fi +} + +#check if supported operating system +checkIfSupportedOS() +{ + getOs + + LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") + + case "$LINUX_DIST_IN_LOWER_CASE" in + *"ubuntu"* ) + echo "INFO: Operating system is Ubuntu." + ;; + *"redhat"* ) + echo "INFO: Operating system is Red Hat." + ;; + *"centos"* ) + echo "INFO: Operating system is CentOS." + ;; + *"darwin"* ) + #if the OS is mac then exit + logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." + exit 1 + ;; + * ) + logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." + while true; do + read -p "Would you like to continue anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + exit 1 + ;; + * ) echo "Please answer yes or no.";; + esac + done + ;; + esac +} + +getOs() +{ + # Determine OS platform + UNAME=$(uname | tr "[:upper:]" "[:lower:]") + # If Linux, try to determine specific distribution + if [ "$UNAME" == "linux" ]; then + # If available, use LSB to identify distribution + if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then + LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) + # Otherwise, use release info file + else + LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) + fi + fi + + # For everything else (or if above failed), just use generic identifier + if [ "$LINUX_DIST" == "" ]; then + LINUX_DIST=$(uname) + fi +} + +#sets linux variables which will be used across various functions +setLinuxVariables() +{ + #set host name + HOST_NAME=$(hostname) + + #set loggly account url + LOGGLY_ACCOUNT_URL=https://$LOGGLY_ACCOUNT.loggly.com +} + +#checks if all the various endpoints used for configuring loggly are accessible +checkIfLogglyServersAccessible() +{ + echo "INFO: Checking if $LOGGLY_ACCOUNT_URL is reachable." + if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then + echo "INFO: $LOGGLY_ACCOUNT_URL is reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_ACCOUNT_URL is not reachable. Please check your network and firewall settings." + exit 1 + fi + + echo "INFO: Checking if $LOGS_01_HOST is reachable." + if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then + echo "INFO: $LOGS_01_HOST is reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." + exit 1 + fi + + echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." + if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then + echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system." + exit 1 + fi + + echo "INFO: Checking if Gen2 account." + if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "404 NOT FOUND" | wc -l) == 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: This scripts need a Gen2 account. Please contact Loggly support." + exit 1 + else + echo "INFO: It is a Gen2 account." + fi +} + +#check if user name and password is valid +checkIfValidUserNamePassword() +{ + echo "INFO: Checking if provided username and password is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." + exit 1 + else + logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." + fi +} + +getAuthToken() +{ + if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." + #get authentication token if user has not provided one + tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") + + #get the string from index 0 to first occurence of , + tokenstr=${tokenstr%%,*} + + #get the string from index 0 to last occurence of " + tokenstr=${tokenstr%\"*} + + #get the string from first occurence of " to the end + tokenstr=${tokenstr#*\"} + + LOGGLY_AUTH_TOKEN=$tokenstr + + logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" + fi +} + +#check if authentication token is valid +checkIfValidAuthToken() +{ + echo "INFO: Checking if provided auth token is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." + else + logMsgToConfigSysLog "ERROR" "ERROR: Invalid authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." + exit 1 + fi +} + +#check if rsyslog is configured as service. If it is configured as service and not started, start the service +checkIfRsyslogConfiguredAsService() +{ + if [ -f /etc/init.d/$RSYSLOG_SERVICE ]; then + logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." + else + logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." + exit 1 + fi + + if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." + sudo service $RSYSLOG_SERVICE start + fi +} + + +#check if multiple versions of rsyslog is configured +checkIfMultipleRsyslogConfigured() +{ + if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." + exit 1 + fi +} + +#check if mimimum version of rsyslog required to configure loggly is met +checkIfMinVersionOfRsyslog() +{ + RSYSLOG_VERSION=$(sudo $RSYSLOGD -version | grep "$RSYSLOGD") + RSYSLOG_VERSION=${RSYSLOG_VERSION#* } + RSYSLOG_VERSION=${RSYSLOG_VERSION%,*} + RSYSLOG_VERSION=$RSYSLOG_VERSION | tr -d " " + if [ $(compareVersions $RSYSLOG_VERSION $MIN_RSYSLOG_VERSION 3) -lt 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Min rsyslog version required is 5.8.0." + exit 1 + fi +} + +#check if SeLinux service is enforced +checkIfSelinuxServiceEnforced() +{ + isSelinuxInstalled=$(getenforce -ds 2>/dev/null) + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: selinux status is not enforced." + elif [ $(sudo getenforce | grep "Enforcing" | wc -l) -gt 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: selinux status is 'Enforcing'. Please disable it and start the rsyslog daemon manually." + exit 1 + fi +} + +#write 22-loggly,conf file to /etc/rsyslog.d directory after checking with user if override is needed +write22LogglyConfFile() +{ + echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist." + if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." + while true; do + read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + checkAuthTokenAndWriteContents; + break;; + [Nn]* ) + LINUX_DO_VERIFICATION="false" + logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE" + checkAuthTokenAndWriteContents + fi +} + +#check if authentication token is valid and then write contents to 22-loggly.conf file to /etc/rsyslog.d directory +checkAuthTokenAndWriteContents() +{ + if [ "$LOGGLY_ACCOUNT" != "" ]; then + writeContents $LOGGLY_ACCOUNT $LOGGLY_AUTH_TOKEN $LOGGLY_DISTRIBUTION_ID $LOGS_01_HOST $LOGGLY_SYSLOG_PORT + restartRsyslog + else + logMsgToConfigSysLog "ERROR" "ERROR: Loggly auth token is required to configure rsyslog. Please pass -a while running script." + exit 1 + fi +} + +#write the contents to 22-loggly.conf file +writeContents() +{ +inputStr=" +# ------------------------------------------------------- +# Syslog Logging Directives for Loggly ($1.loggly.com) +# ------------------------------------------------------- + +# Define the template used for sending logs to Loggly. Do not change this format. +\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\" + +# Send messages to Loggly over TCP using the template. +*.* @@$4:$5;LogglyFormat + +# ------------------------------------------------------- +# End of Syslog Logging Directives for Loggly +# ------------------------------------------------------- +" +sudo cat << EOIPFW >> $LOGGLY_RSYSLOG_CONFFILE +$inputStr +EOIPFW +} + +#create /var/spool/rsyslog directory if not already present. Modify the permission of this directory for Ubuntu +createRsyslogDir() +{ + if [ -d "$RSYSLOG_DIR" ]; then + logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_DIR already exist, so not creating directory." + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + logMsgToConfigSysLog "INFO" "INFO: Changing the permission on the rsyslog in /var/spool" + sudo chown -R syslog:adm $RSYSLOG_DIR + fi + else + logMsgToConfigSysLog "INFO" "INFO: Creating directory $SYSLOGDIR" + sudo mkdir -v $RSYSLOG_DIR + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + sudo chown -R syslog:adm $RSYSLOG_DIR + fi + fi +} + +#check if the logs made it to Loggly +checkIfLogsMadeToLoggly() +{ + logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly." + uuid=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) + + queryParam="syslog.appName%3ALOGGLYVERIFY%20$uuid" + logger -t "LOGGLYVERIFY" "LOGGLYVERIFY-Test message for verification with UUID $uuid" + + counter=1 + maxCounter=10 + finalCount=0 + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + + while [ "$finalCount" -eq 0 ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" + logMsgToConfigSysLog "ERROR" "ERROR: Verification logs did not make it to Loggly in time. Please check your token & network/firewall settings and retry." + exit 1 + fi + done + + if [ "$finalCount" -eq 1 ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." + if [ "$IS_INVOKED" = "" ]; then + exit 0 + fi + fi + +} + +#delete 22-loggly.conf file +remove22LogglyConfFile() +{ + if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then + sudo rm -rf "$LOGGLY_RSYSLOG_CONFFILE" + fi +} + +#compares two version numbers, used for comparing versions of various softwares +compareVersions () +{ + typeset IFS='.' + typeset -a v1=( $1 ) + typeset -a v2=( $2 ) + typeset n diff + + for (( n=0; n<$3; n+=1 )); do + diff=$((v1[n]-v2[n])) + if [ $diff -ne 0 ] ; then + [ $diff -le 0 ] && echo '-1' || echo '1' + return + fi + done + echo '0' +} + +#restart rsyslog +restartRsyslog() +{ + logMsgToConfigSysLog "INFO" "INFO: Restarting the $RSYSLOG_SERVICE service." + sudo service $RSYSLOG_SERVICE restart + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: $RSYSLOG_SERVICE did not restart gracefully. Please restart $RSYSLOG_SERVICE manually." + fi +} + +#logs message to config syslog +logMsgToConfigSysLog() +{ + #$1 variable will be SUCCESS or ERROR or INFO or WARNING + #$2 variable will be the message + cslStatus=$1 + cslMessage=$2 + echo "$cslMessage" + currentTime=$(date) + + #for Linux system, we need to use -d switch to decode base64 whereas + #for Mac system, we need to use -D switch to decode + varUname=$(uname) + if [[ $varUname == 'Linux' ]]; then + enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) + elif [[ $varUname == 'Darwin' ]]; then + enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) + fi + + if [ $? -ne 0 ]; then + echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." + exit 1 + fi + + sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" + + #if it is an error, then log message "Script Failed" to config syslog and exit the script + if [[ $cslStatus == "ERROR" ]]; then + sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" + echo $MANUAL_CONFIG_INSTRUCTION + exit 1 + fi + + #if it is a success, then log message "Script Succeeded" to config syslog and exit the script + if [[ $cslStatus == "SUCCESS" ]]; then + sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" + fi +} + +#payload construction to send log to config syslog +sendPayloadToConfigSysLog() +{ + if [ "$APP_TAG" = "" ]; then + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}" + else + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}" + fi + curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 +} + +#$1 return the count of records in loggly, $2 is the query param to search in loggly +searchAndFetch() +{ + URL_WITH_SPACE_CHARS=$2 + + #replacing all the spaces with %20 + url="${URL_WITH_SPACE_CHARS//\ /%20}" + + result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") + + if [ -z "$result" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." + exit 1 + fi + id=$(echo "$result" | grep -v "{" | grep id | awk '{print $2}') + # strip last double quote from id + id="${id%\"}" + # strip first double quote from id + id="${id#\"}" + url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" + + # retrieve the data + result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") + count=$(echo "$result" | grep total_events | awk '{print $2}') + count="${count%\,}" + eval $1="'$count'" + if [ "$count" -gt 0 ]; then + timestamp=$(echo "$result" | grep timestamp) + fi +} + +#get password in the form of asterisk +getPassword() +{ + unset LOGGLY_PASSWORD + prompt="Please enter Loggly Password:" + while IFS= read -p "$prompt" -r -s -n 1 char + do + if [[ $char == $'\0' ]] + then + break + fi + prompt='*' + LOGGLY_PASSWORD+="$char" + done + echo +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-linux [-a loggly auth account or subdomain] [-r to remove] +usage: configure-linux [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## +if [ "$1" != "being-invoked" ]; then + if [ $# -eq 0 ]; then + usage + exit + else + while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --remove ) + LOGGLY_REMOVE="true" + ;; + -h | --help) + usage + exit + ;; + *) usage + exit + ;; + esac + shift + done + fi + + if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then + removeLogglyConf + elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConf + else + usage + fi +else + IS_INVOKED="true" +fi + +########## Get Inputs from User - End ########## diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh new file mode 100644 index 0000000..2165a19 --- /dev/null +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -0,0 +1,419 @@ +#!/bin/bash + +#downloads configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh & configure-file-monitoring.sh" +#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-file-monitoring.sh +source configure-file-monitoring.sh "being-invoked" + +########## Variable Declarations - Start ########## +#name of the current script +SCRIPT_NAME=configure-s3-file-monitoring.sh +#version of the current script +SCRIPT_VERSION=1.0 + +#s3 bucket name to configure +LOGGLY_S3_BUCKET_NAME= + +#s3 bucket file to configure +LOGGLY_S3_FILE_NAME= + +#alias name, will be used as tag & state file name etc. provided by user +LOGGLY_S3_ALIAS= + +#file alias provided by the user +APP_TAG="\"s3file-alias\":\"\"" + +#name and location of syslog file +FILE_SYSLOG_CONFFILE= + +#name and location of syslog backup file +FILE_SYSLOG_CONFFILE_BACKUP= + +#holds variable if any of the file is configured +IS_ANY_FILE_CONFIGURED="false" + +#value for temp directory +TEMP_DIR= + +#time when the rsyslog is started and we want to monitor if the logs made to loggly +START_DATE_TIME= + +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" + +########## Variable Declarations - End ########## + +# executing the script for loggly to install and configure syslog +installLogglyConfForFile() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating configure Loggly for file monitoring." + + #check if the provided alias is correct or not + checkIfValidAliasForInvoke + + #check if the linux environment is compatible for Loggly + checkLinuxLogglyCompatibility + + #check if s3cmd utility is installed and configured + checkIfS3cmdInstalledAndConfigured + + #check if s3bucket is valid + checkIfValidS3Bucket + + #check if s3bucket file is valid + checkIfValidS3FileName + + #configure loggly for Linux + installLogglyConf + + #create temporary directory + createTempDir + + #download S3 files from bucket to temp directory + downloadS3FilesFromBucketToTempDir + + #download S3 file to temp directory + downloadS3FilesToTempDir + + #invoke file monitoring on each file after checking if it is a text file or not + invokeFileMonitoring + + if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then + #check if s3 logs made it to loggly + checkIfS3LogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." + else + logMsgToConfigSysLog "WARN" "WARN: Did not find any files to configure. Nothing to do." + fi + + #delete temporary directory + deleteTempDir +} + + +#executing script to remove loggly configuration for File +removeLogglyConfForFile() +{ + logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #check if alias provided is the correct one + checkIfValidAliasForRevoke + + #remove file monitoring + removeFileMonitoring + + #log success message + logMsgToConfigSysLog "INFO" "INFO: Rollback completed." +} + +checkIfValidAliasForInvoke() +{ + if ls $RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf &> /dev/null; then + logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_S3_ALIAS is already taken. Please try with another one." + exit 1 + fi +} + +checkIfS3cmdInstalledAndConfigured() +{ + if hash s3cmd 2>/dev/null; then + var=$(s3cmd ls 2>/dev/null) + if [ "$var" != "" ]; then + logMsgToConfigSysLog "INFO" "INFO: s3cmd is already configured on your system" + else + logMsgToConfigSysLog "INFO" "INFO: s3cmd is not configured on your system. Trying to configure." + configureS3cmd + fi + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not present on your system. Setting it up on your system" + downloadAndConfigureS3cmd + fi +} + +downloadAndConfigureS3cmd() +{ + #download and install s3cmd + case "$LINUX_DIST" in + *"Ubuntu"* ) + sudo apt-get install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Ubuntu" ; exit 1; } + ;; + *"RedHat"* ) + sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Red Hat. Please ensure you have EPEL installed." ; exit 1; } + ;; + *"CentOS"* ) + sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on CentOS. Please ensure you have EPEL installed." ; exit 1; } + ;; + * ) + ;; + esac + + configureS3cmd +} + +configureS3cmd() +{ + #configure s3cmd + s3cmd --configure + + #check if s3cmd configured successfully now + if [ $(s3cmd > /dev/null 2>&1 | grep "ERROR: Missing command. Please run with --help for more information." | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" + exit 1 + fi +} + +checkIfValidS3Bucket() +{ + if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then + logMsgToConfigSysLog "INFO" "INFO: Check if valid S3 Bucket name." + sudo s3cmd ls -r $LOGGLY_S3_BUCKET_NAME > /dev/null 2>&1 || { logMsgToConfigSysLog "ERROR" "ERROR: Invalid S3 Bucket name" ; exit 1; } + fi +} + +checkIfValidS3FileName() +{ + if [ "$LOGGLY_S3_FILE_NAME" != "" ]; then + logMsgToConfigSysLog "INFO" "INFO: Check if valid S3 file name." + sudo s3cmd ls $LOGGLY_S3_FILE_NAME > /dev/null 2>&1 || { logMsgToConfigSysLog "ERROR" "ERROR: Invalid S3 File name" ; exit 1; } + fi +} + +createTempDir() +{ + TEMP_DIR=/tmp/$LOGGLY_S3_ALIAS + if [ -d "$TEMP_DIR" ]; then + if [ "$(ls -A $TEMP_DIR)" ]; then + logMsgToConfigSysLog "WARN" "WARN: There are some files/folders already present in $TEMP_DIR. If you continue, the files currently inside the $TEMP_DIR will also be configured to send logs to loggly." + while true; do + read -p "Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Discontinuing with s3 file monitoring configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + fi + else + mkdir /tmp/$LOGGLY_S3_ALIAS + fi +} + +downloadS3FilesFromBucketToTempDir() +{ + if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then + #Files are downloaded in nested directory + cd $TEMP_DIR + echo "Downloading files, may take some time..." + s3cmd get -r -f $LOGGLY_S3_BUCKET_NAME > /dev/null 2>&1 + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Error downloading files recursively from $LOGGLY_S3_BUCKET_NAME" + exit 1 + fi + fi +} + +downloadS3FilesToTempDir() +{ + if [ "$LOGGLY_S3_FILE_NAME" != "" ]; then + cd $TEMP_DIR + echo "Downloading file, may take some time..." + s3cmd get -f $LOGGLY_S3_FILE_NAME > /dev/null 2>&1 + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Error downloading file $LOGGLY_S3_FILE_NAME" + exit 1 + fi + fi +} + +invokeFileMonitoring() +{ + dir=/tmp/$LOGGLY_S3_ALIAS + #TODO: Not supporting multiple files with same name in different directories + #only supporting file with naming convention *.* + for f in $(find $dir -name '*.*') + do + fileNameWithExt=${f##*/} + uniqueFileName=$(echo "$fileNameWithExt" | tr . _) + var=$(file $f) + + if [ ${var##*\ } == "text" -o ${var##*\ } == "Text" ]; then + + LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$LOGGLY_S3_ALIAS + LOGGLY_FILE_TO_MONITOR=$f + constructFileVariables + checkLogFileSize $LOGGLY_FILE_TO_MONITOR + write21ConfFileContents + IS_ANY_FILE_CONFIGURED="true" + + else + logMsgToConfigSysLog "WARN" "WARN: File $fileNameWithExt is not a text file. Ignoring." + fi + done + + if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then + restartRsyslog + START_DATE_TIME=`date +"%F %T.%3N"` + echo "Start time found: $START_DATE_TIME" + fi +} + +deleteTempDir() +{ + rm -fr $TEMP_DIR +} + +checkIfS3LogsMadeToLoggly() +{ + #LOGGLY_S3_ALIAS + #for alias in "${field[@]}" + #do + # LOGGLY_FILE_TO_MONITOR_ALIAS="$alias" + # checkIfFileLogsMadeToLoggly + #done + counter=1 + maxCounter=10 + + fileInitialLogCount=0 + fileLatestLogCount=0 + #queryParam="syslog.appName%3A%2A$LOGGLY_S3_ALIAS&from=$START_DATE_TIME&until=now&size=1" + queryParam="syslog.appName%3A%2A$LOGGLY_S3_ALIAS&from=-15m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: ${queryUrl//\ /%20}" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the logs made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + + #checking if the s3 logs from start_time to current_time have made to the loggly + searchAndFetch fileLatestLogCount "$queryUrl" + let counter=$counter+1 + + while [ "$fileLatestLogCount" -le "$fileInitialLogCount" ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch fileLatestLogCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + exit 1 + fi + done + + if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "" ]; then + exit 0 + fi + fi +} + +checkIfValidAliasForRevoke() +{ + if ! ls $RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf &> /dev/null; then + #logMsgToConfigSysLog "INFO" "INFO: $LOGGLY_S3_ALIAS found." + #else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_S3_ALIAS does not exist. Please provide the correct s3 alias." + exit 1 + fi +} + +removeFileMonitoring() +{ + FILES=$RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf + for f in $FILES + do + aliasName=${f##*/} + aliasName=${aliasName%.*} + aliasName=${aliasName#21-filemonitoring-} + + LOGGLY_FILE_TO_MONITOR_ALIAS=$aliasName + constructFileVariables + remove21ConfFile + done + restartRsyslog +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3b s3bucketname or -s3f s3filename] [-s3l s3alias] +usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] +usage: configure-s3-file-monitoring [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## + +if [ $# -eq 0 ]; then + usage + exit +else +while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --rollback ) + LOGGLY_ROLLBACK="true" + ;; + -s3b | --s3bucketname ) shift + LOGGLY_S3_BUCKET_NAME=$1 + echo "S3 Bucket Name: $LOGGLY_S3_BUCKET_NAME" + ;; + -s3f | --s3filename ) shift + LOGGLY_S3_FILE_NAME=$1 + echo "S3 File Name: $LOGGLY_S3_FILE_NAME" + ;; + -s3l | --s3alias ) shift + LOGGLY_S3_ALIAS=$1 + echo "File alias: $LOGGLY_S3_ALIAS" + ;; + -h | --help) + usage + exit + ;; + esac + shift +done +fi + +if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_S3_ALIAS" != "" -a \( "$LOGGLY_S3_BUCKET_NAME" != "" -o "$LOGGLY_S3_FILE_NAME" != "" \) ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConfForFile +elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_S3_ALIAS" != "" ]; then + removeLogglyConfForFile +else + usage +fi +########## Get Inputs from User - End ########## \ No newline at end of file From f9c864da68010c0753c81b1b00305a2b5a6e0cc7 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Sun, 27 Jul 2014 10:46:10 -0700 Subject: [PATCH 47/95] Add support to Amazon AMI OS --- Linux Script/configure-linux.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 83d5ad5..8982ae5 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -203,6 +203,9 @@ checkIfSupportedOS() *"centos"* ) echo "INFO: Operating system is CentOS." ;; + *"amazon"* ) + echo "INFO: Operating system is Amazon AMI." + ;; *"darwin"* ) #if the OS is mac then exit logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." @@ -234,6 +237,9 @@ getOs() # If available, use LSB to identify distribution if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) + # If system-release is available, then try to identify the name + elif [ -f /etc/system-release ]; then + LINUX_DIST=$(cat /etc/*-release | cut -f 1 -d " ") # Otherwise, use release info file else LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) @@ -602,8 +608,12 @@ sendPayloadToConfigSysLog() #$1 return the count of records in loggly, $2 is the query param to search in loggly searchAndFetch() { - url=$2 + URL_WITH_SPACE_CHARS=$2 + + #replacing all the spaces with %20 + url="${URL_WITH_SPACE_CHARS//\ /%20}" result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") + if [ -z "$result" ]; then logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." exit 1 @@ -622,7 +632,7 @@ searchAndFetch() eval $1="'$count'" if [ "$count" -gt 0 ]; then timestamp=$(echo "$result" | grep timestamp) - fi + fi } #get password in the form of asterisk From 8fee9389064b5d8fc8175077634760880e22469b Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 28 Jul 2014 09:53:19 +0530 Subject: [PATCH 48/95] Removed extra variable --- Linux Script/configure-linux.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 8982ae5..6125517 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -608,10 +608,8 @@ sendPayloadToConfigSysLog() #$1 return the count of records in loggly, $2 is the query param to search in loggly searchAndFetch() { - URL_WITH_SPACE_CHARS=$2 + url=$2 - #replacing all the spaces with %20 - url="${URL_WITH_SPACE_CHARS//\ /%20}" result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") if [ -z "$result" ]; then From 2cac672032457cdb23bb440577d45db2fa367323 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 28 Jul 2014 09:55:33 +0530 Subject: [PATCH 49/95] Updated command --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 6125517..404ea02 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -239,7 +239,7 @@ getOs() LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) # If system-release is available, then try to identify the name elif [ -f /etc/system-release ]; then - LINUX_DIST=$(cat /etc/*-release | cut -f 1 -d " ") + LINUX_DIST=$(cat /etc/system-release | cut -f 1 -d " ") # Otherwise, use release info file else LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) From 6c54ab797f2a38d5851a4f59277e5cf8cbe4f848 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 28 Jul 2014 18:27:27 +0530 Subject: [PATCH 50/95] Updated version number --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 404ea02..1774fec 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.3 #application tag. This will get overwritten by the child script which calls this APP_TAG= From c64f4b3e37b6ff20acf060a08688903e0820706a Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 28 Jul 2014 18:28:34 +0530 Subject: [PATCH 51/95] Updated version number --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b0e234c..482a197 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.01 +SCRIPT_VERSION=1.1 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -345,4 +345,4 @@ else usage fi -########## Get Inputs from User - End ########## \ No newline at end of file +########## Get Inputs from User - End ########## From e7a73286dd28c3573a15c67d20adbcc7d27021e1 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 28 Jul 2014 18:28:58 +0530 Subject: [PATCH 52/95] Updated version number --- Modular Scripts/Tomcat/configure-tomcat.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index bc92fbe..af2a1b3 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-tomcat.sh #version of the current script -SCRIPT_VERSION=1.01 +SCRIPT_VERSION=1.1 #minimum version of tomcat to enable log rotation MIN_TOMCAT_VERSION=6.0.33.0 @@ -704,4 +704,4 @@ else usage fi -########## Get Inputs from User - End ########## \ No newline at end of file +########## Get Inputs from User - End ########## From 3d44e13951b17eed269a3c2132c4eebd77be6a72 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 28 Jul 2014 18:33:33 +0530 Subject: [PATCH 53/95] Updated Scripts (Under development) --- .../configure-file-monitoring.sh | 8 +- .../S3Logs Monitoring/configure-linux.sh | 13 +-- .../configure-s3-file-monitoring.sh | 79 ++++++++----------- 3 files changed, 46 insertions(+), 54 deletions(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh index 5482aaf..771f621 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh @@ -1,7 +1,7 @@ #!/bin/bash #downloads configure-linux.sh -echo "INFO: Downloading dependencies - configure-linux.sh" +#echo "INFO: Downloading dependencies - configure-linux.sh" #curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh source configure-linux.sh "being-invoked" @@ -278,10 +278,12 @@ checkIfConfFileExist() #remove 21.conf file remove21ConfFile() { - echo "INFO: Deleting the loggly file syslog conf file." + echo "INFO: Deleting the loggly syslog conf file $FILE_SYSLOG_CONFFILE." if [ -f "$FILE_SYSLOG_CONFFILE" ]; then sudo rm -rf "$FILE_SYSLOG_CONFFILE" - echo "INFO: Removed all the modified files." + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then + echo "INFO: Removed all the modified files." + fi else logMsgToConfigSysLog "WARN" "WARN: $FILE_SYSLOG_CONFFILE file was not found." fi diff --git a/Modular Scripts/S3Logs Monitoring/configure-linux.sh b/Modular Scripts/S3Logs Monitoring/configure-linux.sh index 234a623..404ea02 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-linux.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-linux.sh @@ -203,6 +203,9 @@ checkIfSupportedOS() *"centos"* ) echo "INFO: Operating system is CentOS." ;; + *"amazon"* ) + echo "INFO: Operating system is Amazon AMI." + ;; *"darwin"* ) #if the OS is mac then exit logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." @@ -234,6 +237,9 @@ getOs() # If available, use LSB to identify distribution if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) + # If system-release is available, then try to identify the name + elif [ -f /etc/system-release ]; then + LINUX_DIST=$(cat /etc/system-release | cut -f 1 -d " ") # Otherwise, use release info file else LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) @@ -602,10 +608,7 @@ sendPayloadToConfigSysLog() #$1 return the count of records in loggly, $2 is the query param to search in loggly searchAndFetch() { - URL_WITH_SPACE_CHARS=$2 - - #replacing all the spaces with %20 - url="${URL_WITH_SPACE_CHARS//\ /%20}" + url=$2 result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") @@ -627,7 +630,7 @@ searchAndFetch() eval $1="'$count'" if [ "$count" -gt 0 ]; then timestamp=$(echo "$result" | grep timestamp) - fi + fi } #get password in the form of asterisk diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 2165a19..a52fdcb 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -36,21 +36,18 @@ IS_ANY_FILE_CONFIGURED="false" #value for temp directory TEMP_DIR= -#time when the rsyslog is started and we want to monitor if the logs made to loggly -START_DATE_TIME= - MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog -installLogglyConfForFile() +installLogglyConfForS3() { #log message indicating starting of Loggly configuration logMsgToConfigSysLog "INFO" "INFO: Initiating configure Loggly for file monitoring." #check if the provided alias is correct or not - checkIfValidAliasForInvoke + checkIfS3AliasAlreadyTaken #check if the linux environment is compatible for Loggly checkLinuxLogglyCompatibility @@ -62,7 +59,7 @@ installLogglyConfForFile() checkIfValidS3Bucket #check if s3bucket file is valid - checkIfValidS3FileName + checkIfValidS3File #configure loggly for Linux installLogglyConf @@ -71,20 +68,17 @@ installLogglyConfForFile() createTempDir #download S3 files from bucket to temp directory - downloadS3FilesFromBucketToTempDir + downloadS3Bucket #download S3 file to temp directory - downloadS3FilesToTempDir + downloadS3File #invoke file monitoring on each file after checking if it is a text file or not - invokeFileMonitoring + invokeS3FileMonitoring if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then #check if s3 logs made it to loggly checkIfS3LogsMadeToLoggly - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." else logMsgToConfigSysLog "WARN" "WARN: Did not find any files to configure. Nothing to do." fi @@ -94,8 +88,8 @@ installLogglyConfForFile() } -#executing script to remove loggly configuration for File -removeLogglyConfForFile() +#executing script to remove loggly configuration for S3 files +removeLogglyConfForS3() { logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." @@ -106,16 +100,16 @@ removeLogglyConfForFile() checkIfSupportedOS #check if alias provided is the correct one - checkIfValidAliasForRevoke + checkIfS3AliasExist #remove file monitoring - removeFileMonitoring + removeS3FileMonitoring #log success message logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } -checkIfValidAliasForInvoke() +checkIfS3AliasAlreadyTaken() { if ls $RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf &> /dev/null; then logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_S3_ALIAS is already taken. Please try with another one." @@ -135,11 +129,12 @@ checkIfS3cmdInstalledAndConfigured() fi else logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not present on your system. Setting it up on your system" - downloadAndConfigureS3cmd + downloadS3cmd + configureS3cmd fi } - -downloadAndConfigureS3cmd() + +downloadS3cmd() { #download and install s3cmd case "$LINUX_DIST" in @@ -156,7 +151,6 @@ downloadAndConfigureS3cmd() ;; esac - configureS3cmd } configureS3cmd() @@ -181,7 +175,7 @@ checkIfValidS3Bucket() fi } -checkIfValidS3FileName() +checkIfValidS3File() { if [ "$LOGGLY_S3_FILE_NAME" != "" ]; then logMsgToConfigSysLog "INFO" "INFO: Check if valid S3 file name." @@ -213,7 +207,7 @@ createTempDir() fi } -downloadS3FilesFromBucketToTempDir() +downloadS3Bucket() { if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then #Files are downloaded in nested directory @@ -227,7 +221,7 @@ downloadS3FilesFromBucketToTempDir() fi } -downloadS3FilesToTempDir() +downloadS3File() { if [ "$LOGGLY_S3_FILE_NAME" != "" ]; then cd $TEMP_DIR @@ -240,12 +234,12 @@ downloadS3FilesToTempDir() fi } -invokeFileMonitoring() +invokeS3FileMonitoring() { dir=/tmp/$LOGGLY_S3_ALIAS #TODO: Not supporting multiple files with same name in different directories #only supporting file with naming convention *.* - for f in $(find $dir -name '*.*') + for f in $(find $dir -name '*') do fileNameWithExt=${f##*/} uniqueFileName=$(echo "$fileNameWithExt" | tr . _) @@ -267,8 +261,6 @@ invokeFileMonitoring() if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then restartRsyslog - START_DATE_TIME=`date +"%F %T.%3N"` - echo "Start time found: $START_DATE_TIME" fi } @@ -279,26 +271,19 @@ deleteTempDir() checkIfS3LogsMadeToLoggly() { - #LOGGLY_S3_ALIAS - #for alias in "${field[@]}" - #do - # LOGGLY_FILE_TO_MONITOR_ALIAS="$alias" - # checkIfFileLogsMadeToLoggly - #done counter=1 maxCounter=10 fileInitialLogCount=0 fileLatestLogCount=0 - #queryParam="syslog.appName%3A%2A$LOGGLY_S3_ALIAS&from=$START_DATE_TIME&until=now&size=1" - queryParam="syslog.appName%3A%2A$LOGGLY_S3_ALIAS&from=-15m&until=now&size=1" + queryParam="syslog.appName%3A%2A$LOGGLY_S3_ALIAS&from=-5m&until=now&size=1" + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" - logMsgToConfigSysLog "INFO" "INFO: Search URL: ${queryUrl//\ /%20}" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" logMsgToConfigSysLog "INFO" "INFO: Verifying if the logs made it to Loggly." logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." - - #checking if the s3 logs from start_time to current_time have made to the loggly + #get the final count of file logs for past 5 minutes searchAndFetch fileLatestLogCount "$queryUrl" let counter=$counter+1 @@ -316,14 +301,15 @@ checkIfS3LogsMadeToLoggly() done if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." - if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "" ]; then - exit 0 + if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_S3_BUCKET_NAME bucket logs to Loggly." + else + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_S3_FILE_NAME logs to Loggly." fi fi } -checkIfValidAliasForRevoke() +checkIfS3AliasExist() { if ! ls $RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf &> /dev/null; then #logMsgToConfigSysLog "INFO" "INFO: $LOGGLY_S3_ALIAS found." @@ -333,7 +319,7 @@ checkIfValidAliasForRevoke() fi } -removeFileMonitoring() +removeS3FileMonitoring() { FILES=$RSYSLOG_ETCDIR_CONF/*$LOGGLY_S3_ALIAS.conf for f in $FILES @@ -346,6 +332,7 @@ removeFileMonitoring() constructFileVariables remove21ConfFile done + echo "INFO: Removed all the modified files." restartRsyslog } @@ -410,9 +397,9 @@ if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_S3_ALIAS" ! if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi - installLogglyConfForFile + installLogglyConfForS3 elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_S3_ALIAS" != "" ]; then - removeLogglyConfForFile + removeLogglyConfForS3 else usage fi From beccd2aefdc296e700c16ce8598990427aeb44d1 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 29 Jul 2014 18:49:58 +0530 Subject: [PATCH 54/95] Added check for sub domain --- Linux Script/configure-linux.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 1774fec..7dc16ad 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.3 +SCRIPT_VERSION=1.4 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -265,14 +265,6 @@ setLinuxVariables() #checks if all the various endpoints used for configuring loggly are accessible checkIfLogglyServersAccessible() { - echo "INFO: Checking if $LOGGLY_ACCOUNT_URL is reachable." - if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then - echo "INFO: $LOGGLY_ACCOUNT_URL is reachable." - else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_ACCOUNT_URL is not reachable. Please check your network and firewall settings." - exit 1 - fi - echo "INFO: Checking if $LOGS_01_HOST is reachable." if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then echo "INFO: $LOGS_01_HOST is reachable." @@ -289,6 +281,14 @@ checkIfLogglyServersAccessible() exit 1 fi + echo "INFO: Checking if \"$LOGGLY_ACCOUNT\" subdomain is valid." + if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then + echo "INFO: $LOGGLY_ACCOUNT_URL is valid and reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: This is not a recognized subdomain. Please ask the account owner for the subdomain they signed up with." + exit 1 + fi + echo "INFO: Checking if Gen2 account." if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "404 NOT FOUND" | wc -l) == 1 ]; then logMsgToConfigSysLog "ERROR" "ERROR: This scripts need a Gen2 account. Please contact Loggly support." @@ -303,8 +303,8 @@ checkIfValidUserNamePassword() { echo "INFO: Checking if provided username and password is correct." if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." - exit 1 + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. You may check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" + exit 1 else logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." fi @@ -569,7 +569,7 @@ logMsgToConfigSysLog() #for Mac system, we need to use -D switch to decode varUname=$(uname) if [[ $varUname == 'Linux' ]]; then - enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) + enabler=$(echo -n MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) elif [[ $varUname == 'Darwin' ]]; then enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) fi From 9d522f8e376c58ca8ed026efed51a1e315327fcb Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 30 Jul 2014 20:50:21 +0530 Subject: [PATCH 55/95] Changed message if Mac OSX found --- Linux Script/configure-linux.sh | 11 +- .../configure-file-monitoring.sh | 6 +- .../S3Logs Monitoring/configure-linux.sh | 715 ------------------ .../configure-s3-file-monitoring.sh | 73 +- 4 files changed, 51 insertions(+), 754 deletions(-) delete mode 100644 Modular Scripts/S3Logs Monitoring/configure-linux.sh diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 7dc16ad..bfa0a7f 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.4 +SCRIPT_VERSION=1.5 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -208,7 +208,7 @@ checkIfSupportedOS() ;; *"darwin"* ) #if the OS is mac then exit - logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." + logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs" exit 1 ;; * ) @@ -303,7 +303,8 @@ checkIfValidUserNamePassword() { echo "INFO: Checking if provided username and password is correct." if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. You may check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" + logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." exit 1 else logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." @@ -584,7 +585,9 @@ logMsgToConfigSysLog() #if it is an error, then log message "Script Failed" to config syslog and exit the script if [[ $cslStatus == "ERROR" ]]; then sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" - echo $MANUAL_CONFIG_INSTRUCTION + if [ "$varUname" != "Darwin" ]; then + echo $MANUAL_CONFIG_INSTRUCTION + fi exit 1 fi diff --git a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh index 771f621..95cd14f 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh @@ -1,15 +1,15 @@ #!/bin/bash #downloads configure-linux.sh -#echo "INFO: Downloading dependencies - configure-linux.sh" -#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh" +curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh source configure-linux.sh "being-invoked" ########## Variable Declarations - Start ########## #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.2 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= diff --git a/Modular Scripts/S3Logs Monitoring/configure-linux.sh b/Modular Scripts/S3Logs Monitoring/configure-linux.sh deleted file mode 100644 index 404ea02..0000000 --- a/Modular Scripts/S3Logs Monitoring/configure-linux.sh +++ /dev/null @@ -1,715 +0,0 @@ -#!/bin/bash - -#trapping Control + C -#these statements must be the first statements in the script to trap the CTRL C event - -trap ctrl_c INT - -function ctrl_c() { - logMsgToConfigSysLog "INFO" "INFO: Aborting the script." - exit 1 -} - -########## Variable Declarations - Start ########## - -#name of the current script. This will get overwritten by the child script which calls this -SCRIPT_NAME=configure-linux.sh -#version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.1 - -#application tag. This will get overwritten by the child script which calls this -APP_TAG= - -#directory location for syslog -RSYSLOG_ETCDIR_CONF=/etc/rsyslog.d -#name and location of loggly syslog file -LOGGLY_RSYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/22-loggly.conf -#name and location of loggly syslog backup file -LOGGLY_RSYSLOG_CONFFILE_BACKUP=$LOGGLY_RSYSLOG_CONFFILE.loggly.bk - -#syslog directory -RSYSLOG_DIR=/var/spool/rsyslog -#rsyslog service name -RSYSLOG_SERVICE=rsyslog -#rsyslogd -RSYSLOGD=rsyslogd -#minimum version of rsyslog to enable logging to loggly -MIN_RSYSLOG_VERSION=5.8.0 -#this variable will hold the users syslog version -RSYSLOG_VERSION= - -#this variable will hold the host name -HOST_NAME= -#this variable will hold the name of the linux distribution -LINUX_DIST= - -#host name for logs-01.loggly.com -LOGS_01_HOST=logs-01.loggly.com -LOGS_01_URL=https://$LOGS_01_HOST -#this variable will contain loggly account url in the format https://$LOGGLY_ACCOUNT.loggly.com -LOGGLY_ACCOUNT_URL= -#loggly.com URL -LOGGLY_COM_URL=https://www.loggly.com - -######Inputs provided by user###### -#this variable will hold the loggly account name provided by user. -#this is a mandatory input -LOGGLY_ACCOUNT= -#this variable will hold the loggly authentication token provided by user. -#this is a mandatory input -LOGGLY_AUTH_TOKEN= -#this variable will identify if the user has selected to rollback settings -LOGGLY_ROLLBACK= -#this variable will hold the user name provided by user -#this is a mandatory input -LOGGLY_USERNAME= -#this variable will hold the password provided by user -#this is a mandatory input -LOGGLY_PASSWORD= - -#variables used in 22-loggly.conf file -LOGGLY_SYSLOG_PORT=514 -LOGGLY_DISTRIBUTION_ID="41058" - -#Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this -#on how to configure the child application -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." - -#this variable is set if the script is invoked via some other calling script -IS_INVOKED= - -#this variable will hold if the check env function for linux is invoked -LINUX_ENV_VALIDATED="false" - -#this variable will inform if verification needs to be performed -LINUX_DO_VERIFICATION="true" - -########## Variable Declarations - End ########## - -#check if the Linux environment is compatible with Loggly. -#Also set few variables after the check. -checkLinuxLogglyCompatibility() -{ - #check if the user has root permission to run this script - checkIfUserHasRootPrivileges - - #check if the OS is supported by the script. If no, then exit - checkIfSupportedOS - - #set the basic variables needed by this script - setLinuxVariables - - #check if the Loggly servers are accessible. If no, ask user to check network connectivity & exit - checkIfLogglyServersAccessible - - #check if user credentials are valid. If no, then exit - checkIfValidUserNamePassword - - #get authentication token if not provided - getAuthToken - - #check if authentication token is valid. If no, then exit. - checkIfValidAuthToken - - #check if rsyslog is configured as service. If no, then exit - checkIfRsyslogConfiguredAsService - - #check if multiple rsyslog are present in the system. If yes, then exit - checkIfMultipleRsyslogConfigured - - #check for the minimum version of rsyslog i.e 5.8.0. If no, then exit - checkIfMinVersionOfRsyslog - - #check if selinux service is enforced. if yes, ask the user to manually disable and exit the script - checkIfSelinuxServiceEnforced - - LINUX_ENV_VALIDATED="true" -} - -# executing the script for loggly to install and configure rsyslog. -installLogglyConf() -{ - #log message indicating starting of Loggly configuration - logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." - - if [ "$LINUX_ENV_VALIDATED" = "false" ]; then - checkLinuxLogglyCompatibility - fi - - #if all the above check passes, write the 22-loggly.conf file - write22LogglyConfFile - - #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu - createRsyslogDir - - if [ "$LINUX_DO_VERIFICATION" = "true" ]; then - #check if the logs are going to loggly fro linux system now - checkIfLogsMadeToLoggly - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." - fi -} - -#remove loggly configuration from Linux system -removeLogglyConf() -{ - #log message indicating starting of Loggly configuration - logMsgToConfigSysLog "INFO" "INFO: Initiating uninstall Loggly for Linux." - - #check if the user has root permission to run this script - checkIfUserHasRootPrivileges - - #check if the OS is supported by the script. If no, then exit - checkIfSupportedOS - - #set the basic variables needed by this script - setLinuxVariables - - #remove 22-loggly.conf file - remove22LogglyConfFile - - #restart rsyslog service - restartRsyslog - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Uninstalled Loggly configuration from Linux system." -} - -#checks if user has root privileges -checkIfUserHasRootPrivileges() -{ - #This script needs to be run as a sudo user - if [[ $EUID -ne 0 ]]; then - logMsgToConfigSysLog "ERROR" "ERROR: This script must be run as root." - exit 1 - fi -} - -#check if supported operating system -checkIfSupportedOS() -{ - getOs - - LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") - - case "$LINUX_DIST_IN_LOWER_CASE" in - *"ubuntu"* ) - echo "INFO: Operating system is Ubuntu." - ;; - *"redhat"* ) - echo "INFO: Operating system is Red Hat." - ;; - *"centos"* ) - echo "INFO: Operating system is CentOS." - ;; - *"amazon"* ) - echo "INFO: Operating system is Amazon AMI." - ;; - *"darwin"* ) - #if the OS is mac then exit - logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." - exit 1 - ;; - * ) - logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." - while true; do - read -p "Would you like to continue anyway? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - exit 1 - ;; - * ) echo "Please answer yes or no.";; - esac - done - ;; - esac -} - -getOs() -{ - # Determine OS platform - UNAME=$(uname | tr "[:upper:]" "[:lower:]") - # If Linux, try to determine specific distribution - if [ "$UNAME" == "linux" ]; then - # If available, use LSB to identify distribution - if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then - LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) - # If system-release is available, then try to identify the name - elif [ -f /etc/system-release ]; then - LINUX_DIST=$(cat /etc/system-release | cut -f 1 -d " ") - # Otherwise, use release info file - else - LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) - fi - fi - - # For everything else (or if above failed), just use generic identifier - if [ "$LINUX_DIST" == "" ]; then - LINUX_DIST=$(uname) - fi -} - -#sets linux variables which will be used across various functions -setLinuxVariables() -{ - #set host name - HOST_NAME=$(hostname) - - #set loggly account url - LOGGLY_ACCOUNT_URL=https://$LOGGLY_ACCOUNT.loggly.com -} - -#checks if all the various endpoints used for configuring loggly are accessible -checkIfLogglyServersAccessible() -{ - echo "INFO: Checking if $LOGGLY_ACCOUNT_URL is reachable." - if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then - echo "INFO: $LOGGLY_ACCOUNT_URL is reachable." - else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_ACCOUNT_URL is not reachable. Please check your network and firewall settings." - exit 1 - fi - - echo "INFO: Checking if $LOGS_01_HOST is reachable." - if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then - echo "INFO: $LOGS_01_HOST is reachable." - else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." - exit 1 - fi - - echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." - if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then - echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." - else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system." - exit 1 - fi - - echo "INFO: Checking if Gen2 account." - if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "404 NOT FOUND" | wc -l) == 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: This scripts need a Gen2 account. Please contact Loggly support." - exit 1 - else - echo "INFO: It is a Gen2 account." - fi -} - -#check if user name and password is valid -checkIfValidUserNamePassword() -{ - echo "INFO: Checking if provided username and password is correct." - if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." - exit 1 - else - logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." - fi -} - -getAuthToken() -{ - if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then - logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." - #get authentication token if user has not provided one - tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") - - #get the string from index 0 to first occurence of , - tokenstr=${tokenstr%%,*} - - #get the string from index 0 to last occurence of " - tokenstr=${tokenstr%\"*} - - #get the string from first occurence of " to the end - tokenstr=${tokenstr#*\"} - - LOGGLY_AUTH_TOKEN=$tokenstr - - logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" - fi -} - -#check if authentication token is valid -checkIfValidAuthToken() -{ - echo "INFO: Checking if provided auth token is correct." - if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then - logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." - else - logMsgToConfigSysLog "ERROR" "ERROR: Invalid authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." - exit 1 - fi -} - -#check if rsyslog is configured as service. If it is configured as service and not started, start the service -checkIfRsyslogConfiguredAsService() -{ - if [ -f /etc/init.d/$RSYSLOG_SERVICE ]; then - logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." - else - logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." - exit 1 - fi - - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then - logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." - sudo service $RSYSLOG_SERVICE start - fi -} - - -#check if multiple versions of rsyslog is configured -checkIfMultipleRsyslogConfigured() -{ - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." - exit 1 - fi -} - -#check if mimimum version of rsyslog required to configure loggly is met -checkIfMinVersionOfRsyslog() -{ - RSYSLOG_VERSION=$(sudo $RSYSLOGD -version | grep "$RSYSLOGD") - RSYSLOG_VERSION=${RSYSLOG_VERSION#* } - RSYSLOG_VERSION=${RSYSLOG_VERSION%,*} - RSYSLOG_VERSION=$RSYSLOG_VERSION | tr -d " " - if [ $(compareVersions $RSYSLOG_VERSION $MIN_RSYSLOG_VERSION 3) -lt 0 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Min rsyslog version required is 5.8.0." - exit 1 - fi -} - -#check if SeLinux service is enforced -checkIfSelinuxServiceEnforced() -{ - isSelinuxInstalled=$(getenforce -ds 2>/dev/null) - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "INFO" "INFO: selinux status is not enforced." - elif [ $(sudo getenforce | grep "Enforcing" | wc -l) -gt 0 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: selinux status is 'Enforcing'. Please disable it and start the rsyslog daemon manually." - exit 1 - fi -} - -#write 22-loggly,conf file to /etc/rsyslog.d directory after checking with user if override is needed -write22LogglyConfFile() -{ - echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist." - if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; - sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; - checkAuthTokenAndWriteContents; - break;; - [Nn]* ) - LINUX_DO_VERIFICATION="false" - logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." - break;; - * ) echo "Please answer yes or no.";; - esac - done - else - logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE" - checkAuthTokenAndWriteContents - fi -} - -#check if authentication token is valid and then write contents to 22-loggly.conf file to /etc/rsyslog.d directory -checkAuthTokenAndWriteContents() -{ - if [ "$LOGGLY_ACCOUNT" != "" ]; then - writeContents $LOGGLY_ACCOUNT $LOGGLY_AUTH_TOKEN $LOGGLY_DISTRIBUTION_ID $LOGS_01_HOST $LOGGLY_SYSLOG_PORT - restartRsyslog - else - logMsgToConfigSysLog "ERROR" "ERROR: Loggly auth token is required to configure rsyslog. Please pass -a while running script." - exit 1 - fi -} - -#write the contents to 22-loggly.conf file -writeContents() -{ -inputStr=" -# ------------------------------------------------------- -# Syslog Logging Directives for Loggly ($1.loggly.com) -# ------------------------------------------------------- - -# Define the template used for sending logs to Loggly. Do not change this format. -\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\" - -# Send messages to Loggly over TCP using the template. -*.* @@$4:$5;LogglyFormat - -# ------------------------------------------------------- -# End of Syslog Logging Directives for Loggly -# ------------------------------------------------------- -" -sudo cat << EOIPFW >> $LOGGLY_RSYSLOG_CONFFILE -$inputStr -EOIPFW -} - -#create /var/spool/rsyslog directory if not already present. Modify the permission of this directory for Ubuntu -createRsyslogDir() -{ - if [ -d "$RSYSLOG_DIR" ]; then - logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_DIR already exist, so not creating directory." - if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then - logMsgToConfigSysLog "INFO" "INFO: Changing the permission on the rsyslog in /var/spool" - sudo chown -R syslog:adm $RSYSLOG_DIR - fi - else - logMsgToConfigSysLog "INFO" "INFO: Creating directory $SYSLOGDIR" - sudo mkdir -v $RSYSLOG_DIR - if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then - sudo chown -R syslog:adm $RSYSLOG_DIR - fi - fi -} - -#check if the logs made it to Loggly -checkIfLogsMadeToLoggly() -{ - logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly." - uuid=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) - - queryParam="syslog.appName%3ALOGGLYVERIFY%20$uuid" - logger -t "LOGGLYVERIFY" "LOGGLYVERIFY-Test message for verification with UUID $uuid" - - counter=1 - maxCounter=10 - finalCount=0 - - queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" - logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" - - logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." - logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." - searchAndFetch finalCount "$queryUrl" - let counter=$counter+1 - - while [ "$finalCount" -eq 0 ]; do - echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." - sleep 30 - echo "INFO: Done waiting. Verifying again." - logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." - searchAndFetch finalCount "$queryUrl" - let counter=$counter+1 - if [ "$counter" -gt "$maxCounter" ]; then - MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" - logMsgToConfigSysLog "ERROR" "ERROR: Verification logs did not make it to Loggly in time. Please check your token & network/firewall settings and retry." - exit 1 - fi - done - - if [ "$finalCount" -eq 1 ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." - if [ "$IS_INVOKED" = "" ]; then - exit 0 - fi - fi - -} - -#delete 22-loggly.conf file -remove22LogglyConfFile() -{ - if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then - sudo rm -rf "$LOGGLY_RSYSLOG_CONFFILE" - fi -} - -#compares two version numbers, used for comparing versions of various softwares -compareVersions () -{ - typeset IFS='.' - typeset -a v1=( $1 ) - typeset -a v2=( $2 ) - typeset n diff - - for (( n=0; n<$3; n+=1 )); do - diff=$((v1[n]-v2[n])) - if [ $diff -ne 0 ] ; then - [ $diff -le 0 ] && echo '-1' || echo '1' - return - fi - done - echo '0' -} - -#restart rsyslog -restartRsyslog() -{ - logMsgToConfigSysLog "INFO" "INFO: Restarting the $RSYSLOG_SERVICE service." - sudo service $RSYSLOG_SERVICE restart - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: $RSYSLOG_SERVICE did not restart gracefully. Please restart $RSYSLOG_SERVICE manually." - fi -} - -#logs message to config syslog -logMsgToConfigSysLog() -{ - #$1 variable will be SUCCESS or ERROR or INFO or WARNING - #$2 variable will be the message - cslStatus=$1 - cslMessage=$2 - echo "$cslMessage" - currentTime=$(date) - - #for Linux system, we need to use -d switch to decode base64 whereas - #for Mac system, we need to use -D switch to decode - varUname=$(uname) - if [[ $varUname == 'Linux' ]]; then - enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) - elif [[ $varUname == 'Darwin' ]]; then - enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) - fi - - if [ $? -ne 0 ]; then - echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." - exit 1 - fi - - sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" - - #if it is an error, then log message "Script Failed" to config syslog and exit the script - if [[ $cslStatus == "ERROR" ]]; then - sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" - echo $MANUAL_CONFIG_INSTRUCTION - exit 1 - fi - - #if it is a success, then log message "Script Succeeded" to config syslog and exit the script - if [[ $cslStatus == "SUCCESS" ]]; then - sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" - fi -} - -#payload construction to send log to config syslog -sendPayloadToConfigSysLog() -{ - if [ "$APP_TAG" = "" ]; then - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}" - else - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}" - fi - curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 -} - -#$1 return the count of records in loggly, $2 is the query param to search in loggly -searchAndFetch() -{ - url=$2 - - result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") - - if [ -z "$result" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." - exit 1 - fi - id=$(echo "$result" | grep -v "{" | grep id | awk '{print $2}') - # strip last double quote from id - id="${id%\"}" - # strip first double quote from id - id="${id#\"}" - url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" - - # retrieve the data - result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") - count=$(echo "$result" | grep total_events | awk '{print $2}') - count="${count%\,}" - eval $1="'$count'" - if [ "$count" -gt 0 ]; then - timestamp=$(echo "$result" | grep timestamp) - fi -} - -#get password in the form of asterisk -getPassword() -{ - unset LOGGLY_PASSWORD - prompt="Please enter Loggly Password:" - while IFS= read -p "$prompt" -r -s -n 1 char - do - if [[ $char == $'\0' ]] - then - break - fi - prompt='*' - LOGGLY_PASSWORD+="$char" - done - echo -} - -#display usage syntax -usage() -{ -cat << EOF -usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] -usage: configure-linux [-a loggly auth account or subdomain] [-r to remove] -usage: configure-linux [-h for help] -EOF -} - -########## Get Inputs from User - Start ########## -if [ "$1" != "being-invoked" ]; then - if [ $# -eq 0 ]; then - usage - exit - else - while [ "$1" != "" ]; do - case $1 in - -t | --token ) shift - LOGGLY_AUTH_TOKEN=$1 - echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" - ;; - -a | --account ) shift - LOGGLY_ACCOUNT=$1 - echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" - ;; - -u | --username ) shift - LOGGLY_USERNAME=$1 - echo "Username is set" - ;; - -p | --password ) shift - LOGGLY_PASSWORD=$1 - ;; - -r | --remove ) - LOGGLY_REMOVE="true" - ;; - -h | --help) - usage - exit - ;; - *) usage - exit - ;; - esac - shift - done - fi - - if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then - removeLogglyConf - elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then - if [ "$LOGGLY_PASSWORD" = "" ]; then - getPassword - fi - installLogglyConf - else - usage - fi -else - IS_INVOKED="true" -fi - -########## Get Inputs from User - End ########## diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index a52fdcb..75c8a04 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -1,9 +1,8 @@ #!/bin/bash -#downloads configure-linux.sh -echo "INFO: Downloading dependencies - configure-linux.sh & configure-file-monitoring.sh" +#downloads configure-file-monitoring.sh +#echo "INFO: Downloading dependencies - configure-file-monitoring.sh" #curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh -#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-file-monitoring.sh source configure-file-monitoring.sh "being-invoked" ########## Variable Declarations - Start ########## @@ -36,6 +35,8 @@ IS_ANY_FILE_CONFIGURED="false" #value for temp directory TEMP_DIR= +IS_S3CMD_CONFIGURED_BY_SCRIPT="false" + MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" ########## Variable Declarations - End ########## @@ -120,51 +121,59 @@ checkIfS3AliasAlreadyTaken() checkIfS3cmdInstalledAndConfigured() { if hash s3cmd 2>/dev/null; then - var=$(s3cmd ls 2>/dev/null) - if [ "$var" != "" ]; then + checkIfS3cmdConfigured + else + logMsgToConfigSysLog "INFO" "INFO: s3cmd is not present on your system. Setting it up on your system" + downloadS3cmd + configureS3cmd + fi +} + +checkIfS3cmdConfigured() +{ + var=$(s3cmd ls 2>/dev/null) + if [ "$var" != "" ]; then + if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then logMsgToConfigSysLog "INFO" "INFO: s3cmd is already configured on your system" else + logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" + fi + else + if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then logMsgToConfigSysLog "INFO" "INFO: s3cmd is not configured on your system. Trying to configure." configureS3cmd + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" + exit 1 fi - else - logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not present on your system. Setting it up on your system" - downloadS3cmd - configureS3cmd - fi + fi } - + downloadS3cmd() { #download and install s3cmd - case "$LINUX_DIST" in - *"Ubuntu"* ) - sudo apt-get install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Ubuntu" ; exit 1; } - ;; - *"RedHat"* ) - sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Red Hat. Please ensure you have EPEL installed." ; exit 1; } - ;; - *"CentOS"* ) - sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on CentOS. Please ensure you have EPEL installed." ; exit 1; } - ;; - * ) - ;; - esac - + + #checking if the Linux is yum based or apt-get based + YUM_BASED=$(command -v yum) + APT_GET_BASED=$(command -v apt-get) + + if [ "$YUM_BASED" != "" ]; then + sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST. Please ensure you have EPEL installed." ; exit 1; } + elif [ "$APT_GET_BASED" != "" ]; then + sudo apt-get install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." ; exit 1; } + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." + exit 1 + fi } configureS3cmd() { #configure s3cmd s3cmd --configure - + IS_S3CMD_CONFIGURED_BY_SCRIPT="true" #check if s3cmd configured successfully now - if [ $(s3cmd > /dev/null 2>&1 | grep "ERROR: Missing command. Please run with --help for more information." | wc -l) == 1 ]; then - logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" - else - logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" - exit 1 - fi + checkIfS3cmdConfigured } checkIfValidS3Bucket() From 151b89e893594d80b20f44f67bdaaf8f0faaf999 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 30 Jul 2014 20:50:31 +0530 Subject: [PATCH 56/95] Revert "Changed message if Mac OSX found" This reverts commit 9d522f8e376c58ca8ed026efed51a1e315327fcb. --- Linux Script/configure-linux.sh | 11 +- .../configure-file-monitoring.sh | 6 +- .../S3Logs Monitoring/configure-linux.sh | 715 ++++++++++++++++++ .../configure-s3-file-monitoring.sh | 73 +- 4 files changed, 754 insertions(+), 51 deletions(-) create mode 100644 Modular Scripts/S3Logs Monitoring/configure-linux.sh diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index bfa0a7f..7dc16ad 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.5 +SCRIPT_VERSION=1.4 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -208,7 +208,7 @@ checkIfSupportedOS() ;; *"darwin"* ) #if the OS is mac then exit - logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs" + logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." exit 1 ;; * ) @@ -303,8 +303,7 @@ checkIfValidUserNamePassword() { echo "INFO: Checking if provided username and password is correct." if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then - logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. You may check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" exit 1 else logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." @@ -585,9 +584,7 @@ logMsgToConfigSysLog() #if it is an error, then log message "Script Failed" to config syslog and exit the script if [[ $cslStatus == "ERROR" ]]; then sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" - if [ "$varUname" != "Darwin" ]; then - echo $MANUAL_CONFIG_INSTRUCTION - fi + echo $MANUAL_CONFIG_INSTRUCTION exit 1 fi diff --git a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh index 95cd14f..771f621 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh @@ -1,15 +1,15 @@ #!/bin/bash #downloads configure-linux.sh -echo "INFO: Downloading dependencies - configure-linux.sh" -curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +#echo "INFO: Downloading dependencies - configure-linux.sh" +#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh source configure-linux.sh "being-invoked" ########## Variable Declarations - Start ########## #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.2 +SCRIPT_VERSION=1.1 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= diff --git a/Modular Scripts/S3Logs Monitoring/configure-linux.sh b/Modular Scripts/S3Logs Monitoring/configure-linux.sh new file mode 100644 index 0000000..404ea02 --- /dev/null +++ b/Modular Scripts/S3Logs Monitoring/configure-linux.sh @@ -0,0 +1,715 @@ +#!/bin/bash + +#trapping Control + C +#these statements must be the first statements in the script to trap the CTRL C event + +trap ctrl_c INT + +function ctrl_c() { + logMsgToConfigSysLog "INFO" "INFO: Aborting the script." + exit 1 +} + +########## Variable Declarations - Start ########## + +#name of the current script. This will get overwritten by the child script which calls this +SCRIPT_NAME=configure-linux.sh +#version of the current script. This will get overwritten by the child script which calls this +SCRIPT_VERSION=1.1 + +#application tag. This will get overwritten by the child script which calls this +APP_TAG= + +#directory location for syslog +RSYSLOG_ETCDIR_CONF=/etc/rsyslog.d +#name and location of loggly syslog file +LOGGLY_RSYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/22-loggly.conf +#name and location of loggly syslog backup file +LOGGLY_RSYSLOG_CONFFILE_BACKUP=$LOGGLY_RSYSLOG_CONFFILE.loggly.bk + +#syslog directory +RSYSLOG_DIR=/var/spool/rsyslog +#rsyslog service name +RSYSLOG_SERVICE=rsyslog +#rsyslogd +RSYSLOGD=rsyslogd +#minimum version of rsyslog to enable logging to loggly +MIN_RSYSLOG_VERSION=5.8.0 +#this variable will hold the users syslog version +RSYSLOG_VERSION= + +#this variable will hold the host name +HOST_NAME= +#this variable will hold the name of the linux distribution +LINUX_DIST= + +#host name for logs-01.loggly.com +LOGS_01_HOST=logs-01.loggly.com +LOGS_01_URL=https://$LOGS_01_HOST +#this variable will contain loggly account url in the format https://$LOGGLY_ACCOUNT.loggly.com +LOGGLY_ACCOUNT_URL= +#loggly.com URL +LOGGLY_COM_URL=https://www.loggly.com + +######Inputs provided by user###### +#this variable will hold the loggly account name provided by user. +#this is a mandatory input +LOGGLY_ACCOUNT= +#this variable will hold the loggly authentication token provided by user. +#this is a mandatory input +LOGGLY_AUTH_TOKEN= +#this variable will identify if the user has selected to rollback settings +LOGGLY_ROLLBACK= +#this variable will hold the user name provided by user +#this is a mandatory input +LOGGLY_USERNAME= +#this variable will hold the password provided by user +#this is a mandatory input +LOGGLY_PASSWORD= + +#variables used in 22-loggly.conf file +LOGGLY_SYSLOG_PORT=514 +LOGGLY_DISTRIBUTION_ID="41058" + +#Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this +#on how to configure the child application +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." + +#this variable is set if the script is invoked via some other calling script +IS_INVOKED= + +#this variable will hold if the check env function for linux is invoked +LINUX_ENV_VALIDATED="false" + +#this variable will inform if verification needs to be performed +LINUX_DO_VERIFICATION="true" + +########## Variable Declarations - End ########## + +#check if the Linux environment is compatible with Loggly. +#Also set few variables after the check. +checkLinuxLogglyCompatibility() +{ + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setLinuxVariables + + #check if the Loggly servers are accessible. If no, ask user to check network connectivity & exit + checkIfLogglyServersAccessible + + #check if user credentials are valid. If no, then exit + checkIfValidUserNamePassword + + #get authentication token if not provided + getAuthToken + + #check if authentication token is valid. If no, then exit. + checkIfValidAuthToken + + #check if rsyslog is configured as service. If no, then exit + checkIfRsyslogConfiguredAsService + + #check if multiple rsyslog are present in the system. If yes, then exit + checkIfMultipleRsyslogConfigured + + #check for the minimum version of rsyslog i.e 5.8.0. If no, then exit + checkIfMinVersionOfRsyslog + + #check if selinux service is enforced. if yes, ask the user to manually disable and exit the script + checkIfSelinuxServiceEnforced + + LINUX_ENV_VALIDATED="true" +} + +# executing the script for loggly to install and configure rsyslog. +installLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." + + if [ "$LINUX_ENV_VALIDATED" = "false" ]; then + checkLinuxLogglyCompatibility + fi + + #if all the above check passes, write the 22-loggly.conf file + write22LogglyConfFile + + #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu + createRsyslogDir + + if [ "$LINUX_DO_VERIFICATION" = "true" ]; then + #check if the logs are going to loggly fro linux system now + checkIfLogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." + fi +} + +#remove loggly configuration from Linux system +removeLogglyConf() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating uninstall Loggly for Linux." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #set the basic variables needed by this script + setLinuxVariables + + #remove 22-loggly.conf file + remove22LogglyConfFile + + #restart rsyslog service + restartRsyslog + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Uninstalled Loggly configuration from Linux system." +} + +#checks if user has root privileges +checkIfUserHasRootPrivileges() +{ + #This script needs to be run as a sudo user + if [[ $EUID -ne 0 ]]; then + logMsgToConfigSysLog "ERROR" "ERROR: This script must be run as root." + exit 1 + fi +} + +#check if supported operating system +checkIfSupportedOS() +{ + getOs + + LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") + + case "$LINUX_DIST_IN_LOWER_CASE" in + *"ubuntu"* ) + echo "INFO: Operating system is Ubuntu." + ;; + *"redhat"* ) + echo "INFO: Operating system is Red Hat." + ;; + *"centos"* ) + echo "INFO: Operating system is CentOS." + ;; + *"amazon"* ) + echo "INFO: Operating system is Amazon AMI." + ;; + *"darwin"* ) + #if the OS is mac then exit + logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." + exit 1 + ;; + * ) + logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." + while true; do + read -p "Would you like to continue anyway? (yes/no)" yn + case $yn in + [Yy]* ) + break;; + [Nn]* ) + exit 1 + ;; + * ) echo "Please answer yes or no.";; + esac + done + ;; + esac +} + +getOs() +{ + # Determine OS platform + UNAME=$(uname | tr "[:upper:]" "[:lower:]") + # If Linux, try to determine specific distribution + if [ "$UNAME" == "linux" ]; then + # If available, use LSB to identify distribution + if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then + LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) + # If system-release is available, then try to identify the name + elif [ -f /etc/system-release ]; then + LINUX_DIST=$(cat /etc/system-release | cut -f 1 -d " ") + # Otherwise, use release info file + else + LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) + fi + fi + + # For everything else (or if above failed), just use generic identifier + if [ "$LINUX_DIST" == "" ]; then + LINUX_DIST=$(uname) + fi +} + +#sets linux variables which will be used across various functions +setLinuxVariables() +{ + #set host name + HOST_NAME=$(hostname) + + #set loggly account url + LOGGLY_ACCOUNT_URL=https://$LOGGLY_ACCOUNT.loggly.com +} + +#checks if all the various endpoints used for configuring loggly are accessible +checkIfLogglyServersAccessible() +{ + echo "INFO: Checking if $LOGGLY_ACCOUNT_URL is reachable." + if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then + echo "INFO: $LOGGLY_ACCOUNT_URL is reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_ACCOUNT_URL is not reachable. Please check your network and firewall settings." + exit 1 + fi + + echo "INFO: Checking if $LOGS_01_HOST is reachable." + if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then + echo "INFO: $LOGS_01_HOST is reachable." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." + exit 1 + fi + + echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." + if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then + echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." + else + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system." + exit 1 + fi + + echo "INFO: Checking if Gen2 account." + if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "404 NOT FOUND" | wc -l) == 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: This scripts need a Gen2 account. Please contact Loggly support." + exit 1 + else + echo "INFO: It is a Gen2 account." + fi +} + +#check if user name and password is valid +checkIfValidUserNamePassword() +{ + echo "INFO: Checking if provided username and password is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." + exit 1 + else + logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." + fi +} + +getAuthToken() +{ + if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." + #get authentication token if user has not provided one + tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") + + #get the string from index 0 to first occurence of , + tokenstr=${tokenstr%%,*} + + #get the string from index 0 to last occurence of " + tokenstr=${tokenstr%\"*} + + #get the string from first occurence of " to the end + tokenstr=${tokenstr#*\"} + + LOGGLY_AUTH_TOKEN=$tokenstr + + logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" + fi +} + +#check if authentication token is valid +checkIfValidAuthToken() +{ + echo "INFO: Checking if provided auth token is correct." + if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." + else + logMsgToConfigSysLog "ERROR" "ERROR: Invalid authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." + exit 1 + fi +} + +#check if rsyslog is configured as service. If it is configured as service and not started, start the service +checkIfRsyslogConfiguredAsService() +{ + if [ -f /etc/init.d/$RSYSLOG_SERVICE ]; then + logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." + else + logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." + exit 1 + fi + + if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." + sudo service $RSYSLOG_SERVICE start + fi +} + + +#check if multiple versions of rsyslog is configured +checkIfMultipleRsyslogConfigured() +{ + if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." + exit 1 + fi +} + +#check if mimimum version of rsyslog required to configure loggly is met +checkIfMinVersionOfRsyslog() +{ + RSYSLOG_VERSION=$(sudo $RSYSLOGD -version | grep "$RSYSLOGD") + RSYSLOG_VERSION=${RSYSLOG_VERSION#* } + RSYSLOG_VERSION=${RSYSLOG_VERSION%,*} + RSYSLOG_VERSION=$RSYSLOG_VERSION | tr -d " " + if [ $(compareVersions $RSYSLOG_VERSION $MIN_RSYSLOG_VERSION 3) -lt 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Min rsyslog version required is 5.8.0." + exit 1 + fi +} + +#check if SeLinux service is enforced +checkIfSelinuxServiceEnforced() +{ + isSelinuxInstalled=$(getenforce -ds 2>/dev/null) + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "INFO" "INFO: selinux status is not enforced." + elif [ $(sudo getenforce | grep "Enforcing" | wc -l) -gt 0 ]; then + logMsgToConfigSysLog "ERROR" "ERROR: selinux status is 'Enforcing'. Please disable it and start the rsyslog daemon manually." + exit 1 + fi +} + +#write 22-loggly,conf file to /etc/rsyslog.d directory after checking with user if override is needed +write22LogglyConfFile() +{ + echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist." + if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." + while true; do + read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + checkAuthTokenAndWriteContents; + break;; + [Nn]* ) + LINUX_DO_VERIFICATION="false" + logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE" + checkAuthTokenAndWriteContents + fi +} + +#check if authentication token is valid and then write contents to 22-loggly.conf file to /etc/rsyslog.d directory +checkAuthTokenAndWriteContents() +{ + if [ "$LOGGLY_ACCOUNT" != "" ]; then + writeContents $LOGGLY_ACCOUNT $LOGGLY_AUTH_TOKEN $LOGGLY_DISTRIBUTION_ID $LOGS_01_HOST $LOGGLY_SYSLOG_PORT + restartRsyslog + else + logMsgToConfigSysLog "ERROR" "ERROR: Loggly auth token is required to configure rsyslog. Please pass -a while running script." + exit 1 + fi +} + +#write the contents to 22-loggly.conf file +writeContents() +{ +inputStr=" +# ------------------------------------------------------- +# Syslog Logging Directives for Loggly ($1.loggly.com) +# ------------------------------------------------------- + +# Define the template used for sending logs to Loggly. Do not change this format. +\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\" + +# Send messages to Loggly over TCP using the template. +*.* @@$4:$5;LogglyFormat + +# ------------------------------------------------------- +# End of Syslog Logging Directives for Loggly +# ------------------------------------------------------- +" +sudo cat << EOIPFW >> $LOGGLY_RSYSLOG_CONFFILE +$inputStr +EOIPFW +} + +#create /var/spool/rsyslog directory if not already present. Modify the permission of this directory for Ubuntu +createRsyslogDir() +{ + if [ -d "$RSYSLOG_DIR" ]; then + logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_DIR already exist, so not creating directory." + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + logMsgToConfigSysLog "INFO" "INFO: Changing the permission on the rsyslog in /var/spool" + sudo chown -R syslog:adm $RSYSLOG_DIR + fi + else + logMsgToConfigSysLog "INFO" "INFO: Creating directory $SYSLOGDIR" + sudo mkdir -v $RSYSLOG_DIR + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + sudo chown -R syslog:adm $RSYSLOG_DIR + fi + fi +} + +#check if the logs made it to Loggly +checkIfLogsMadeToLoggly() +{ + logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly." + uuid=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) + + queryParam="syslog.appName%3ALOGGLYVERIFY%20$uuid" + logger -t "LOGGLYVERIFY" "LOGGLYVERIFY-Test message for verification with UUID $uuid" + + counter=1 + maxCounter=10 + finalCount=0 + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + + while [ "$finalCount" -eq 0 ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch finalCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" + logMsgToConfigSysLog "ERROR" "ERROR: Verification logs did not make it to Loggly in time. Please check your token & network/firewall settings and retry." + exit 1 + fi + done + + if [ "$finalCount" -eq 1 ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." + if [ "$IS_INVOKED" = "" ]; then + exit 0 + fi + fi + +} + +#delete 22-loggly.conf file +remove22LogglyConfFile() +{ + if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then + sudo rm -rf "$LOGGLY_RSYSLOG_CONFFILE" + fi +} + +#compares two version numbers, used for comparing versions of various softwares +compareVersions () +{ + typeset IFS='.' + typeset -a v1=( $1 ) + typeset -a v2=( $2 ) + typeset n diff + + for (( n=0; n<$3; n+=1 )); do + diff=$((v1[n]-v2[n])) + if [ $diff -ne 0 ] ; then + [ $diff -le 0 ] && echo '-1' || echo '1' + return + fi + done + echo '0' +} + +#restart rsyslog +restartRsyslog() +{ + logMsgToConfigSysLog "INFO" "INFO: Restarting the $RSYSLOG_SERVICE service." + sudo service $RSYSLOG_SERVICE restart + if [ $? -ne 0 ]; then + logMsgToConfigSysLog "WARNING" "WARNING: $RSYSLOG_SERVICE did not restart gracefully. Please restart $RSYSLOG_SERVICE manually." + fi +} + +#logs message to config syslog +logMsgToConfigSysLog() +{ + #$1 variable will be SUCCESS or ERROR or INFO or WARNING + #$2 variable will be the message + cslStatus=$1 + cslMessage=$2 + echo "$cslMessage" + currentTime=$(date) + + #for Linux system, we need to use -d switch to decode base64 whereas + #for Mac system, we need to use -D switch to decode + varUname=$(uname) + if [[ $varUname == 'Linux' ]]; then + enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) + elif [[ $varUname == 'Darwin' ]]; then + enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) + fi + + if [ $? -ne 0 ]; then + echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." + exit 1 + fi + + sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" + + #if it is an error, then log message "Script Failed" to config syslog and exit the script + if [[ $cslStatus == "ERROR" ]]; then + sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" + echo $MANUAL_CONFIG_INSTRUCTION + exit 1 + fi + + #if it is a success, then log message "Script Succeeded" to config syslog and exit the script + if [[ $cslStatus == "SUCCESS" ]]; then + sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" + fi +} + +#payload construction to send log to config syslog +sendPayloadToConfigSysLog() +{ + if [ "$APP_TAG" = "" ]; then + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}" + else + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}" + fi + curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 +} + +#$1 return the count of records in loggly, $2 is the query param to search in loggly +searchAndFetch() +{ + url=$2 + + result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") + + if [ -z "$result" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." + exit 1 + fi + id=$(echo "$result" | grep -v "{" | grep id | awk '{print $2}') + # strip last double quote from id + id="${id%\"}" + # strip first double quote from id + id="${id#\"}" + url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" + + # retrieve the data + result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") + count=$(echo "$result" | grep total_events | awk '{print $2}') + count="${count%\,}" + eval $1="'$count'" + if [ "$count" -gt 0 ]; then + timestamp=$(echo "$result" | grep timestamp) + fi +} + +#get password in the form of asterisk +getPassword() +{ + unset LOGGLY_PASSWORD + prompt="Please enter Loggly Password:" + while IFS= read -p "$prompt" -r -s -n 1 char + do + if [[ $char == $'\0' ]] + then + break + fi + prompt='*' + LOGGLY_PASSWORD+="$char" + done + echo +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-linux [-a loggly auth account or subdomain] [-r to remove] +usage: configure-linux [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## +if [ "$1" != "being-invoked" ]; then + if [ $# -eq 0 ]; then + usage + exit + else + while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --remove ) + LOGGLY_REMOVE="true" + ;; + -h | --help) + usage + exit + ;; + *) usage + exit + ;; + esac + shift + done + fi + + if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then + removeLogglyConf + elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConf + else + usage + fi +else + IS_INVOKED="true" +fi + +########## Get Inputs from User - End ########## diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 75c8a04..a52fdcb 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -1,8 +1,9 @@ #!/bin/bash -#downloads configure-file-monitoring.sh -#echo "INFO: Downloading dependencies - configure-file-monitoring.sh" +#downloads configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh & configure-file-monitoring.sh" #curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-file-monitoring.sh source configure-file-monitoring.sh "being-invoked" ########## Variable Declarations - Start ########## @@ -35,8 +36,6 @@ IS_ANY_FILE_CONFIGURED="false" #value for temp directory TEMP_DIR= -IS_S3CMD_CONFIGURED_BY_SCRIPT="false" - MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" ########## Variable Declarations - End ########## @@ -121,59 +120,51 @@ checkIfS3AliasAlreadyTaken() checkIfS3cmdInstalledAndConfigured() { if hash s3cmd 2>/dev/null; then - checkIfS3cmdConfigured - else - logMsgToConfigSysLog "INFO" "INFO: s3cmd is not present on your system. Setting it up on your system" - downloadS3cmd - configureS3cmd - fi -} - -checkIfS3cmdConfigured() -{ - var=$(s3cmd ls 2>/dev/null) - if [ "$var" != "" ]; then - if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then + var=$(s3cmd ls 2>/dev/null) + if [ "$var" != "" ]; then logMsgToConfigSysLog "INFO" "INFO: s3cmd is already configured on your system" else - logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" - fi - else - if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then logMsgToConfigSysLog "INFO" "INFO: s3cmd is not configured on your system. Trying to configure." configureS3cmd - else - logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" - exit 1 fi - fi + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not present on your system. Setting it up on your system" + downloadS3cmd + configureS3cmd + fi } - + downloadS3cmd() { #download and install s3cmd - - #checking if the Linux is yum based or apt-get based - YUM_BASED=$(command -v yum) - APT_GET_BASED=$(command -v apt-get) - - if [ "$YUM_BASED" != "" ]; then - sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST. Please ensure you have EPEL installed." ; exit 1; } - elif [ "$APT_GET_BASED" != "" ]; then - sudo apt-get install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." ; exit 1; } - else - logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." - exit 1 - fi + case "$LINUX_DIST" in + *"Ubuntu"* ) + sudo apt-get install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Ubuntu" ; exit 1; } + ;; + *"RedHat"* ) + sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Red Hat. Please ensure you have EPEL installed." ; exit 1; } + ;; + *"CentOS"* ) + sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on CentOS. Please ensure you have EPEL installed." ; exit 1; } + ;; + * ) + ;; + esac + } configureS3cmd() { #configure s3cmd s3cmd --configure - IS_S3CMD_CONFIGURED_BY_SCRIPT="true" + #check if s3cmd configured successfully now - checkIfS3cmdConfigured + if [ $(s3cmd > /dev/null 2>&1 | grep "ERROR: Missing command. Please run with --help for more information." | wc -l) == 1 ]; then + logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" + exit 1 + fi } checkIfValidS3Bucket() From cc077958ff60e5dd19f6f0e800aa18c1e5a5f563 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 30 Jul 2014 20:54:18 +0530 Subject: [PATCH 57/95] Changed message if "Darwin OS" --- Linux Script/configure-linux.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 7dc16ad..bfa0a7f 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.4 +SCRIPT_VERSION=1.5 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -208,7 +208,7 @@ checkIfSupportedOS() ;; *"darwin"* ) #if the OS is mac then exit - logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." + logMsgToConfigSysLog "ERROR" "ERROR: This script is for Linux systems, and Darwin or Mac OSX are not currently supported. You can find alternative options here: https://www.loggly.com/docs" exit 1 ;; * ) @@ -303,7 +303,8 @@ checkIfValidUserNamePassword() { echo "INFO: Checking if provided username and password is correct." if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password. You may check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" + logMsgToConfigSysLog "INFO" "INFO: Please check your username or reset your password at $LOGGLY_ACCOUNT_URL/account/users/" + logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." exit 1 else logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." @@ -584,7 +585,9 @@ logMsgToConfigSysLog() #if it is an error, then log message "Script Failed" to config syslog and exit the script if [[ $cslStatus == "ERROR" ]]; then sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" - echo $MANUAL_CONFIG_INSTRUCTION + if [ "$varUname" != "Darwin" ]; then + echo $MANUAL_CONFIG_INSTRUCTION + fi exit 1 fi From 9787d95d9876edbc7ff89776c79f86a24df86bf9 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 30 Jul 2014 20:54:40 +0530 Subject: [PATCH 58/95] Updated s3 Script. Under Developement --- .../configure-file-monitoring.sh | 6 +- .../S3Logs Monitoring/configure-linux.sh | 715 ------------------ .../configure-s3-file-monitoring.sh | 71 +- 3 files changed, 43 insertions(+), 749 deletions(-) delete mode 100644 Modular Scripts/S3Logs Monitoring/configure-linux.sh diff --git a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh index 771f621..95cd14f 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh @@ -1,15 +1,15 @@ #!/bin/bash #downloads configure-linux.sh -#echo "INFO: Downloading dependencies - configure-linux.sh" -#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh" +curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh source configure-linux.sh "being-invoked" ########## Variable Declarations - Start ########## #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.2 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= diff --git a/Modular Scripts/S3Logs Monitoring/configure-linux.sh b/Modular Scripts/S3Logs Monitoring/configure-linux.sh deleted file mode 100644 index 404ea02..0000000 --- a/Modular Scripts/S3Logs Monitoring/configure-linux.sh +++ /dev/null @@ -1,715 +0,0 @@ -#!/bin/bash - -#trapping Control + C -#these statements must be the first statements in the script to trap the CTRL C event - -trap ctrl_c INT - -function ctrl_c() { - logMsgToConfigSysLog "INFO" "INFO: Aborting the script." - exit 1 -} - -########## Variable Declarations - Start ########## - -#name of the current script. This will get overwritten by the child script which calls this -SCRIPT_NAME=configure-linux.sh -#version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.1 - -#application tag. This will get overwritten by the child script which calls this -APP_TAG= - -#directory location for syslog -RSYSLOG_ETCDIR_CONF=/etc/rsyslog.d -#name and location of loggly syslog file -LOGGLY_RSYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/22-loggly.conf -#name and location of loggly syslog backup file -LOGGLY_RSYSLOG_CONFFILE_BACKUP=$LOGGLY_RSYSLOG_CONFFILE.loggly.bk - -#syslog directory -RSYSLOG_DIR=/var/spool/rsyslog -#rsyslog service name -RSYSLOG_SERVICE=rsyslog -#rsyslogd -RSYSLOGD=rsyslogd -#minimum version of rsyslog to enable logging to loggly -MIN_RSYSLOG_VERSION=5.8.0 -#this variable will hold the users syslog version -RSYSLOG_VERSION= - -#this variable will hold the host name -HOST_NAME= -#this variable will hold the name of the linux distribution -LINUX_DIST= - -#host name for logs-01.loggly.com -LOGS_01_HOST=logs-01.loggly.com -LOGS_01_URL=https://$LOGS_01_HOST -#this variable will contain loggly account url in the format https://$LOGGLY_ACCOUNT.loggly.com -LOGGLY_ACCOUNT_URL= -#loggly.com URL -LOGGLY_COM_URL=https://www.loggly.com - -######Inputs provided by user###### -#this variable will hold the loggly account name provided by user. -#this is a mandatory input -LOGGLY_ACCOUNT= -#this variable will hold the loggly authentication token provided by user. -#this is a mandatory input -LOGGLY_AUTH_TOKEN= -#this variable will identify if the user has selected to rollback settings -LOGGLY_ROLLBACK= -#this variable will hold the user name provided by user -#this is a mandatory input -LOGGLY_USERNAME= -#this variable will hold the password provided by user -#this is a mandatory input -LOGGLY_PASSWORD= - -#variables used in 22-loggly.conf file -LOGGLY_SYSLOG_PORT=514 -LOGGLY_DISTRIBUTION_ID="41058" - -#Instruction link on how to configure loggly on linux manually. This will get overwritten by the child script which calls this -#on how to configure the child application -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure rsyslog on Linux are available at https://www.loggly.com/docs/rsyslog-manual-configuration/." - -#this variable is set if the script is invoked via some other calling script -IS_INVOKED= - -#this variable will hold if the check env function for linux is invoked -LINUX_ENV_VALIDATED="false" - -#this variable will inform if verification needs to be performed -LINUX_DO_VERIFICATION="true" - -########## Variable Declarations - End ########## - -#check if the Linux environment is compatible with Loggly. -#Also set few variables after the check. -checkLinuxLogglyCompatibility() -{ - #check if the user has root permission to run this script - checkIfUserHasRootPrivileges - - #check if the OS is supported by the script. If no, then exit - checkIfSupportedOS - - #set the basic variables needed by this script - setLinuxVariables - - #check if the Loggly servers are accessible. If no, ask user to check network connectivity & exit - checkIfLogglyServersAccessible - - #check if user credentials are valid. If no, then exit - checkIfValidUserNamePassword - - #get authentication token if not provided - getAuthToken - - #check if authentication token is valid. If no, then exit. - checkIfValidAuthToken - - #check if rsyslog is configured as service. If no, then exit - checkIfRsyslogConfiguredAsService - - #check if multiple rsyslog are present in the system. If yes, then exit - checkIfMultipleRsyslogConfigured - - #check for the minimum version of rsyslog i.e 5.8.0. If no, then exit - checkIfMinVersionOfRsyslog - - #check if selinux service is enforced. if yes, ask the user to manually disable and exit the script - checkIfSelinuxServiceEnforced - - LINUX_ENV_VALIDATED="true" -} - -# executing the script for loggly to install and configure rsyslog. -installLogglyConf() -{ - #log message indicating starting of Loggly configuration - logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Linux." - - if [ "$LINUX_ENV_VALIDATED" = "false" ]; then - checkLinuxLogglyCompatibility - fi - - #if all the above check passes, write the 22-loggly.conf file - write22LogglyConfFile - - #create rsyslog dir if it doesn't exist, Modify the permission on rsyslog directory if exist on Ubuntu - createRsyslogDir - - if [ "$LINUX_DO_VERIFICATION" = "true" ]; then - #check if the logs are going to loggly fro linux system now - checkIfLogsMadeToLoggly - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." - fi -} - -#remove loggly configuration from Linux system -removeLogglyConf() -{ - #log message indicating starting of Loggly configuration - logMsgToConfigSysLog "INFO" "INFO: Initiating uninstall Loggly for Linux." - - #check if the user has root permission to run this script - checkIfUserHasRootPrivileges - - #check if the OS is supported by the script. If no, then exit - checkIfSupportedOS - - #set the basic variables needed by this script - setLinuxVariables - - #remove 22-loggly.conf file - remove22LogglyConfFile - - #restart rsyslog service - restartRsyslog - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Uninstalled Loggly configuration from Linux system." -} - -#checks if user has root privileges -checkIfUserHasRootPrivileges() -{ - #This script needs to be run as a sudo user - if [[ $EUID -ne 0 ]]; then - logMsgToConfigSysLog "ERROR" "ERROR: This script must be run as root." - exit 1 - fi -} - -#check if supported operating system -checkIfSupportedOS() -{ - getOs - - LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") - - case "$LINUX_DIST_IN_LOWER_CASE" in - *"ubuntu"* ) - echo "INFO: Operating system is Ubuntu." - ;; - *"redhat"* ) - echo "INFO: Operating system is Red Hat." - ;; - *"centos"* ) - echo "INFO: Operating system is CentOS." - ;; - *"amazon"* ) - echo "INFO: Operating system is Amazon AMI." - ;; - *"darwin"* ) - #if the OS is mac then exit - logMsgToConfigSysLog "ERROR" "ERROR: '$LINUX_DIST' operating system is not supported by the script." - exit 1 - ;; - * ) - logMsgToConfigSysLog "WARN" "WARN: The linux distribution '$LINUX_DIST' has not been previously tested with Loggly." - while true; do - read -p "Would you like to continue anyway? (yes/no)" yn - case $yn in - [Yy]* ) - break;; - [Nn]* ) - exit 1 - ;; - * ) echo "Please answer yes or no.";; - esac - done - ;; - esac -} - -getOs() -{ - # Determine OS platform - UNAME=$(uname | tr "[:upper:]" "[:lower:]") - # If Linux, try to determine specific distribution - if [ "$UNAME" == "linux" ]; then - # If available, use LSB to identify distribution - if [ -f /etc/lsb-release -o -d /etc/lsb-release.d ]; then - LINUX_DIST=$(lsb_release -i | cut -d: -f2 | sed s/'^\t'//) - # If system-release is available, then try to identify the name - elif [ -f /etc/system-release ]; then - LINUX_DIST=$(cat /etc/system-release | cut -f 1 -d " ") - # Otherwise, use release info file - else - LINUX_DIST=$(ls -d /etc/[A-Za-z]*[_-][rv]e[lr]* | grep -v "lsb" | cut -d'/' -f3 | cut -d'-' -f1 | cut -d'_' -f1) - fi - fi - - # For everything else (or if above failed), just use generic identifier - if [ "$LINUX_DIST" == "" ]; then - LINUX_DIST=$(uname) - fi -} - -#sets linux variables which will be used across various functions -setLinuxVariables() -{ - #set host name - HOST_NAME=$(hostname) - - #set loggly account url - LOGGLY_ACCOUNT_URL=https://$LOGGLY_ACCOUNT.loggly.com -} - -#checks if all the various endpoints used for configuring loggly are accessible -checkIfLogglyServersAccessible() -{ - echo "INFO: Checking if $LOGGLY_ACCOUNT_URL is reachable." - if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then - echo "INFO: $LOGGLY_ACCOUNT_URL is reachable." - else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGGLY_ACCOUNT_URL is not reachable. Please check your network and firewall settings." - exit 1 - fi - - echo "INFO: Checking if $LOGS_01_HOST is reachable." - if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then - echo "INFO: $LOGS_01_HOST is reachable." - else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." - exit 1 - fi - - echo "INFO: Checking if $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port. This may take some time." - if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then - echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." - else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system." - exit 1 - fi - - echo "INFO: Checking if Gen2 account." - if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "404 NOT FOUND" | wc -l) == 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: This scripts need a Gen2 account. Please contact Loggly support." - exit 1 - else - echo "INFO: It is a Gen2 account." - fi -} - -#check if user name and password is valid -checkIfValidUserNamePassword() -{ - echo "INFO: Checking if provided username and password is correct." - if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep "Unauthorized" | wc -l) == 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid Loggly username or password." - exit 1 - else - logMsgToConfigSysLog "INFO" "INFO: Username and password authorized successfully." - fi -} - -getAuthToken() -{ - if [ "$LOGGLY_AUTH_TOKEN" = "" ]; then - logMsgToConfigSysLog "INFO" "INFO: Authentication token not provided. Trying to retrieve it from $LOGGLY_ACCOUNT_URL account." - #get authentication token if user has not provided one - tokenstr=$(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep -v "token") - - #get the string from index 0 to first occurence of , - tokenstr=${tokenstr%%,*} - - #get the string from index 0 to last occurence of " - tokenstr=${tokenstr%\"*} - - #get the string from first occurence of " to the end - tokenstr=${tokenstr#*\"} - - LOGGLY_AUTH_TOKEN=$tokenstr - - logMsgToConfigSysLog "INFO" "INFO: Retrieved authentication token: $LOGGLY_AUTH_TOKEN" - fi -} - -#check if authentication token is valid -checkIfValidAuthToken() -{ - echo "INFO: Checking if provided auth token is correct." - if [ $(curl -s -u $LOGGLY_USERNAME:$LOGGLY_PASSWORD $LOGGLY_ACCOUNT_URL/apiv2/customer | grep \"$LOGGLY_AUTH_TOKEN\" | wc -l) == 1 ]; then - logMsgToConfigSysLog "INFO" "INFO: Authentication token validated successfully." - else - logMsgToConfigSysLog "ERROR" "ERROR: Invalid authentication token $LOGGLY_AUTH_TOKEN. You can get valid authentication token by following instructions at https://www.loggly.com/docs/customer-token-authentication-token/." - exit 1 - fi -} - -#check if rsyslog is configured as service. If it is configured as service and not started, start the service -checkIfRsyslogConfiguredAsService() -{ - if [ -f /etc/init.d/$RSYSLOG_SERVICE ]; then - logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is present as service." - else - logMsgToConfigSysLog "ERROR" "ERROR: $RSYSLOG_SERVICE is not present as service." - exit 1 - fi - - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -eq 0 ]; then - logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_SERVICE is not running. Attempting to start service." - sudo service $RSYSLOG_SERVICE start - fi -} - - -#check if multiple versions of rsyslog is configured -checkIfMultipleRsyslogConfigured() -{ - if [ $(ps -ef | grep -v grep | grep "$RSYSLOG_SERVICE" | wc -l) -gt 1 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Multiple (more than 1) $RSYSLOG_SERVICE is running." - exit 1 - fi -} - -#check if mimimum version of rsyslog required to configure loggly is met -checkIfMinVersionOfRsyslog() -{ - RSYSLOG_VERSION=$(sudo $RSYSLOGD -version | grep "$RSYSLOGD") - RSYSLOG_VERSION=${RSYSLOG_VERSION#* } - RSYSLOG_VERSION=${RSYSLOG_VERSION%,*} - RSYSLOG_VERSION=$RSYSLOG_VERSION | tr -d " " - if [ $(compareVersions $RSYSLOG_VERSION $MIN_RSYSLOG_VERSION 3) -lt 0 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Min rsyslog version required is 5.8.0." - exit 1 - fi -} - -#check if SeLinux service is enforced -checkIfSelinuxServiceEnforced() -{ - isSelinuxInstalled=$(getenforce -ds 2>/dev/null) - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "INFO" "INFO: selinux status is not enforced." - elif [ $(sudo getenforce | grep "Enforcing" | wc -l) -gt 0 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: selinux status is 'Enforcing'. Please disable it and start the rsyslog daemon manually." - exit 1 - fi -} - -#write 22-loggly,conf file to /etc/rsyslog.d directory after checking with user if override is needed -write22LogglyConfFile() -{ - echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist." - if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; - sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; - checkAuthTokenAndWriteContents; - break;; - [Nn]* ) - LINUX_DO_VERIFICATION="false" - logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." - break;; - * ) echo "Please answer yes or no.";; - esac - done - else - logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE" - checkAuthTokenAndWriteContents - fi -} - -#check if authentication token is valid and then write contents to 22-loggly.conf file to /etc/rsyslog.d directory -checkAuthTokenAndWriteContents() -{ - if [ "$LOGGLY_ACCOUNT" != "" ]; then - writeContents $LOGGLY_ACCOUNT $LOGGLY_AUTH_TOKEN $LOGGLY_DISTRIBUTION_ID $LOGS_01_HOST $LOGGLY_SYSLOG_PORT - restartRsyslog - else - logMsgToConfigSysLog "ERROR" "ERROR: Loggly auth token is required to configure rsyslog. Please pass -a while running script." - exit 1 - fi -} - -#write the contents to 22-loggly.conf file -writeContents() -{ -inputStr=" -# ------------------------------------------------------- -# Syslog Logging Directives for Loggly ($1.loggly.com) -# ------------------------------------------------------- - -# Define the template used for sending logs to Loggly. Do not change this format. -\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$2@$3] %msg%\" - -# Send messages to Loggly over TCP using the template. -*.* @@$4:$5;LogglyFormat - -# ------------------------------------------------------- -# End of Syslog Logging Directives for Loggly -# ------------------------------------------------------- -" -sudo cat << EOIPFW >> $LOGGLY_RSYSLOG_CONFFILE -$inputStr -EOIPFW -} - -#create /var/spool/rsyslog directory if not already present. Modify the permission of this directory for Ubuntu -createRsyslogDir() -{ - if [ -d "$RSYSLOG_DIR" ]; then - logMsgToConfigSysLog "INFO" "INFO: $RSYSLOG_DIR already exist, so not creating directory." - if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then - logMsgToConfigSysLog "INFO" "INFO: Changing the permission on the rsyslog in /var/spool" - sudo chown -R syslog:adm $RSYSLOG_DIR - fi - else - logMsgToConfigSysLog "INFO" "INFO: Creating directory $SYSLOGDIR" - sudo mkdir -v $RSYSLOG_DIR - if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then - sudo chown -R syslog:adm $RSYSLOG_DIR - fi - fi -} - -#check if the logs made it to Loggly -checkIfLogsMadeToLoggly() -{ - logMsgToConfigSysLog "INFO" "INFO: Sending test message to Loggly." - uuid=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) - - queryParam="syslog.appName%3ALOGGLYVERIFY%20$uuid" - logger -t "LOGGLYVERIFY" "LOGGLYVERIFY-Test message for verification with UUID $uuid" - - counter=1 - maxCounter=10 - finalCount=0 - - queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" - logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" - - logMsgToConfigSysLog "INFO" "INFO: Verifying if the log made it to Loggly." - logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." - searchAndFetch finalCount "$queryUrl" - let counter=$counter+1 - - while [ "$finalCount" -eq 0 ]; do - echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." - sleep 30 - echo "INFO: Done waiting. Verifying again." - logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." - searchAndFetch finalCount "$queryUrl" - let counter=$counter+1 - if [ "$counter" -gt "$maxCounter" ]; then - MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" - logMsgToConfigSysLog "ERROR" "ERROR: Verification logs did not make it to Loggly in time. Please check your token & network/firewall settings and retry." - exit 1 - fi - done - - if [ "$finalCount" -eq 1 ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Verification logs successfully transferred to Loggly! You are now sending Linux system logs to Loggly." - if [ "$IS_INVOKED" = "" ]; then - exit 0 - fi - fi - -} - -#delete 22-loggly.conf file -remove22LogglyConfFile() -{ - if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then - sudo rm -rf "$LOGGLY_RSYSLOG_CONFFILE" - fi -} - -#compares two version numbers, used for comparing versions of various softwares -compareVersions () -{ - typeset IFS='.' - typeset -a v1=( $1 ) - typeset -a v2=( $2 ) - typeset n diff - - for (( n=0; n<$3; n+=1 )); do - diff=$((v1[n]-v2[n])) - if [ $diff -ne 0 ] ; then - [ $diff -le 0 ] && echo '-1' || echo '1' - return - fi - done - echo '0' -} - -#restart rsyslog -restartRsyslog() -{ - logMsgToConfigSysLog "INFO" "INFO: Restarting the $RSYSLOG_SERVICE service." - sudo service $RSYSLOG_SERVICE restart - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "WARNING" "WARNING: $RSYSLOG_SERVICE did not restart gracefully. Please restart $RSYSLOG_SERVICE manually." - fi -} - -#logs message to config syslog -logMsgToConfigSysLog() -{ - #$1 variable will be SUCCESS or ERROR or INFO or WARNING - #$2 variable will be the message - cslStatus=$1 - cslMessage=$2 - echo "$cslMessage" - currentTime=$(date) - - #for Linux system, we need to use -d switch to decode base64 whereas - #for Mac system, we need to use -D switch to decode - varUname=$(uname) - if [[ $varUname == 'Linux' ]]; then - enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -d) - elif [[ $varUname == 'Darwin' ]]; then - enabler=$(echo MWVjNGU4ZTEtZmJiMi00N2U3LTkyOWItNzVhMWJmZjVmZmUw | base64 -D) - fi - - if [ $? -ne 0 ]; then - echo "ERROR: Base64 decode is not supported on your Operating System. Please update your system to support Base64." - exit 1 - fi - - sendPayloadToConfigSysLog "$cslStatus" "$cslMessage" "$enabler" - - #if it is an error, then log message "Script Failed" to config syslog and exit the script - if [[ $cslStatus == "ERROR" ]]; then - sendPayloadToConfigSysLog "ERROR" "Script Failed" "$enabler" - echo $MANUAL_CONFIG_INSTRUCTION - exit 1 - fi - - #if it is a success, then log message "Script Succeeded" to config syslog and exit the script - if [[ $cslStatus == "SUCCESS" ]]; then - sendPayloadToConfigSysLog "SUCCESS" "Script Succeeded" "$enabler" - fi -} - -#payload construction to send log to config syslog -sendPayloadToConfigSysLog() -{ - if [ "$APP_TAG" = "" ]; then - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}" - else - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}" - fi - curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 -} - -#$1 return the count of records in loggly, $2 is the query param to search in loggly -searchAndFetch() -{ - url=$2 - - result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") - - if [ -z "$result" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Please check your network/firewall settings & ensure Loggly subdomain, username and password is specified correctly." - exit 1 - fi - id=$(echo "$result" | grep -v "{" | grep id | awk '{print $2}') - # strip last double quote from id - id="${id%\"}" - # strip first double quote from id - id="${id#\"}" - url="$LOGGLY_ACCOUNT_URL/apiv2/events?rsid=$id" - - # retrieve the data - result=$(wget -qO- /dev/null --user "$LOGGLY_USERNAME" --password "$LOGGLY_PASSWORD" "$url") - count=$(echo "$result" | grep total_events | awk '{print $2}') - count="${count%\,}" - eval $1="'$count'" - if [ "$count" -gt 0 ]; then - timestamp=$(echo "$result" | grep timestamp) - fi -} - -#get password in the form of asterisk -getPassword() -{ - unset LOGGLY_PASSWORD - prompt="Please enter Loggly Password:" - while IFS= read -p "$prompt" -r -s -n 1 char - do - if [[ $char == $'\0' ]] - then - break - fi - prompt='*' - LOGGLY_PASSWORD+="$char" - done - echo -} - -#display usage syntax -usage() -{ -cat << EOF -usage: configure-linux [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] -usage: configure-linux [-a loggly auth account or subdomain] [-r to remove] -usage: configure-linux [-h for help] -EOF -} - -########## Get Inputs from User - Start ########## -if [ "$1" != "being-invoked" ]; then - if [ $# -eq 0 ]; then - usage - exit - else - while [ "$1" != "" ]; do - case $1 in - -t | --token ) shift - LOGGLY_AUTH_TOKEN=$1 - echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" - ;; - -a | --account ) shift - LOGGLY_ACCOUNT=$1 - echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" - ;; - -u | --username ) shift - LOGGLY_USERNAME=$1 - echo "Username is set" - ;; - -p | --password ) shift - LOGGLY_PASSWORD=$1 - ;; - -r | --remove ) - LOGGLY_REMOVE="true" - ;; - -h | --help) - usage - exit - ;; - *) usage - exit - ;; - esac - shift - done - fi - - if [ "$LOGGLY_REMOVE" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then - removeLogglyConf - elif [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then - if [ "$LOGGLY_PASSWORD" = "" ]; then - getPassword - fi - installLogglyConf - else - usage - fi -else - IS_INVOKED="true" -fi - -########## Get Inputs from User - End ########## diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index a52fdcb..dd87ba7 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -1,9 +1,8 @@ #!/bin/bash #downloads configure-linux.sh -echo "INFO: Downloading dependencies - configure-linux.sh & configure-file-monitoring.sh" +#echo "INFO: Downloading dependencies - configure-file-monitoring.sh" #curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh -#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-file-monitoring.sh source configure-file-monitoring.sh "being-invoked" ########## Variable Declarations - Start ########## @@ -36,6 +35,8 @@ IS_ANY_FILE_CONFIGURED="false" #value for temp directory TEMP_DIR= +IS_S3CMD_CONFIGURED_BY_SCRIPT="false" + MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" ########## Variable Declarations - End ########## @@ -120,51 +121,59 @@ checkIfS3AliasAlreadyTaken() checkIfS3cmdInstalledAndConfigured() { if hash s3cmd 2>/dev/null; then - var=$(s3cmd ls 2>/dev/null) - if [ "$var" != "" ]; then + checkIfS3cmdConfigured + else + logMsgToConfigSysLog "INFO" "INFO: s3cmd is not present on your system. Setting it up on your system" + downloadS3cmd + configureS3cmd + fi +} + +checkIfS3cmdConfigured() +{ + var=$(s3cmd ls 2>/dev/null) + if [ "$var" != "" ]; then + if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then logMsgToConfigSysLog "INFO" "INFO: s3cmd is already configured on your system" else + logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" + fi + else + if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then logMsgToConfigSysLog "INFO" "INFO: s3cmd is not configured on your system. Trying to configure." configureS3cmd + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" + exit 1 fi - else - logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not present on your system. Setting it up on your system" - downloadS3cmd - configureS3cmd - fi + fi } - + downloadS3cmd() { #download and install s3cmd - case "$LINUX_DIST" in - *"Ubuntu"* ) - sudo apt-get install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Ubuntu" ; exit 1; } - ;; - *"RedHat"* ) - sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on Red Hat. Please ensure you have EPEL installed." ; exit 1; } - ;; - *"CentOS"* ) - sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on CentOS. Please ensure you have EPEL installed." ; exit 1; } - ;; - * ) - ;; - esac - + + #checking if the Linux is yum based or apt-get based + YUM_BASED=$(command -v yum) + APT_GET_BASED=$(command -v apt-get) + + if [ "$YUM_BASED" != "" ]; then + sudo yum install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST. Please ensure you have EPEL installed." ; exit 1; } + elif [ "$APT_GET_BASED" != "" ]; then + sudo apt-get install s3cmd || { logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." ; exit 1; } + else + logMsgToConfigSysLog "ERROR" "ERROR: s3cmd installation failed on $LINUX_DIST." + exit 1 + fi } configureS3cmd() { #configure s3cmd s3cmd --configure - + IS_S3CMD_CONFIGURED_BY_SCRIPT="true" #check if s3cmd configured successfully now - if [ $(s3cmd > /dev/null 2>&1 | grep "ERROR: Missing command. Please run with --help for more information." | wc -l) == 1 ]; then - logMsgToConfigSysLog "INFO" "INFO: s3cmd configured successfully" - else - logMsgToConfigSysLog "ERROR" "ERROR: s3cmd is not configured correctly. Please configure s3cmd using command s3cmd --configure" - exit 1 - fi + checkIfS3cmdConfigured } checkIfValidS3Bucket() From 29d982be636acd68237b301c261bae5cdfa5202d Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 30 Jul 2014 21:16:22 +0530 Subject: [PATCH 59/95] Added comments --- .../S3Logs Monitoring/configure-s3-file-monitoring.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index dd87ba7..435e034 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -118,6 +118,7 @@ checkIfS3AliasAlreadyTaken() fi } +#check if s3cmd utility is installed and configured checkIfS3cmdInstalledAndConfigured() { if hash s3cmd 2>/dev/null; then @@ -129,6 +130,7 @@ checkIfS3cmdInstalledAndConfigured() fi } +#check if s3cmd utility is configured checkIfS3cmdConfigured() { var=$(s3cmd ls 2>/dev/null) @@ -149,10 +151,9 @@ checkIfS3cmdConfigured() fi } +#download and install s3cmd downloadS3cmd() { - #download and install s3cmd - #checking if the Linux is yum based or apt-get based YUM_BASED=$(command -v yum) APT_GET_BASED=$(command -v apt-get) @@ -167,15 +168,16 @@ downloadS3cmd() fi } +#configure s3cmd configureS3cmd() { - #configure s3cmd s3cmd --configure IS_S3CMD_CONFIGURED_BY_SCRIPT="true" #check if s3cmd configured successfully now checkIfS3cmdConfigured } +#check if s3bucket is valid checkIfValidS3Bucket() { if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then From bc505b9d761210ac35faf456c820f322fb1ac9f7 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 31 Jul 2014 18:39:02 +0530 Subject: [PATCH 60/95] Add support to download bucket/ file using -s3b --- .../configure-s3-file-monitoring.sh | 55 ++++++------------- 1 file changed, 16 insertions(+), 39 deletions(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 435e034..6dc00bf 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -14,9 +14,6 @@ SCRIPT_VERSION=1.0 #s3 bucket name to configure LOGGLY_S3_BUCKET_NAME= -#s3 bucket file to configure -LOGGLY_S3_FILE_NAME= - #alias name, will be used as tag & state file name etc. provided by user LOGGLY_S3_ALIAS= @@ -59,9 +56,6 @@ installLogglyConfForS3() #check if s3bucket is valid checkIfValidS3Bucket - #check if s3bucket file is valid - checkIfValidS3File - #configure loggly for Linux installLogglyConf @@ -71,9 +65,6 @@ installLogglyConfForS3() #download S3 files from bucket to temp directory downloadS3Bucket - #download S3 file to temp directory - downloadS3File - #invoke file monitoring on each file after checking if it is a text file or not invokeS3FileMonitoring @@ -182,17 +173,19 @@ checkIfValidS3Bucket() { if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then logMsgToConfigSysLog "INFO" "INFO: Check if valid S3 Bucket name." - sudo s3cmd ls -r $LOGGLY_S3_BUCKET_NAME > /dev/null 2>&1 || { logMsgToConfigSysLog "ERROR" "ERROR: Invalid S3 Bucket name" ; exit 1; } + BUCKET_INFO=$(sudo s3cmd ls -r $LOGGLY_S3_BUCKET_NAME 2>&1) + case $BUCKET_INFO in + ERROR*) + logMsgToConfigSysLog "ERROR" "ERROR: Invalid S3 Bucket name $LOGGLY_S3_BUCKET_NAME." + exit 1 + ;; + "") + logMsgToConfigSysLog "ERROR" "ERROR: No files found in the S3 Bucket $LOGGLY_S3_BUCKET_NAME." + exit 1 + esac fi } -checkIfValidS3File() -{ - if [ "$LOGGLY_S3_FILE_NAME" != "" ]; then - logMsgToConfigSysLog "INFO" "INFO: Check if valid S3 file name." - sudo s3cmd ls $LOGGLY_S3_FILE_NAME > /dev/null 2>&1 || { logMsgToConfigSysLog "ERROR" "ERROR: Invalid S3 File name" ; exit 1; } - fi -} createTempDir() { @@ -232,18 +225,6 @@ downloadS3Bucket() fi } -downloadS3File() -{ - if [ "$LOGGLY_S3_FILE_NAME" != "" ]; then - cd $TEMP_DIR - echo "Downloading file, may take some time..." - s3cmd get -f $LOGGLY_S3_FILE_NAME > /dev/null 2>&1 - if [ $? -ne 0 ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Error downloading file $LOGGLY_S3_FILE_NAME" - exit 1 - fi - fi -} invokeS3FileMonitoring() { @@ -313,9 +294,8 @@ checkIfS3LogsMadeToLoggly() if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_S3_BUCKET_NAME bucket logs to Loggly." - else - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_S3_FILE_NAME logs to Loggly." + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_S3_BUCKET_NAME logs to Loggly." + fi fi } @@ -351,8 +331,8 @@ removeS3FileMonitoring() usage() { cat << EOF -usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3b s3bucketname or -s3f s3filename] [-s3l s3alias] -usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] +usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3b s3bucketname ] [-s3l s3alias] +usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-s3l s3alias] usage: configure-s3-file-monitoring [-h for help] EOF } @@ -387,10 +367,7 @@ while [ "$1" != "" ]; do LOGGLY_S3_BUCKET_NAME=$1 echo "S3 Bucket Name: $LOGGLY_S3_BUCKET_NAME" ;; - -s3f | --s3filename ) shift - LOGGLY_S3_FILE_NAME=$1 - echo "S3 File Name: $LOGGLY_S3_FILE_NAME" - ;; + -s3l | --s3alias ) shift LOGGLY_S3_ALIAS=$1 echo "File alias: $LOGGLY_S3_ALIAS" @@ -404,7 +381,7 @@ while [ "$1" != "" ]; do done fi -if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_S3_ALIAS" != "" -a \( "$LOGGLY_S3_BUCKET_NAME" != "" -o "$LOGGLY_S3_FILE_NAME" != "" \) ]; then +if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_S3_ALIAS" != "" -a \( "$LOGGLY_S3_BUCKET_NAME" != "" \) ]; then if [ "$LOGGLY_PASSWORD" = "" ]; then getPassword fi From cfb14352c11260c58a40c6a2b13b3f746c6f222c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 31 Jul 2014 18:47:25 +0530 Subject: [PATCH 61/95] Minor Bug Fix --- .../S3Logs Monitoring/configure-s3-file-monitoring.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 6dc00bf..a4ede21 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -182,6 +182,7 @@ checkIfValidS3Bucket() "") logMsgToConfigSysLog "ERROR" "ERROR: No files found in the S3 Bucket $LOGGLY_S3_BUCKET_NAME." exit 1 + ;; esac fi } @@ -391,4 +392,4 @@ elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_S3_ALIAS" else usage fi -########## Get Inputs from User - End ########## \ No newline at end of file +########## Get Inputs from User - End ########## From dcab5508d2947bb05a9ed0390bea030c31cbf5c1 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 1 Aug 2014 21:35:15 +0530 Subject: [PATCH 62/95] Updated s3b to s3url and Minor bug fixes --- .../configure-s3-file-monitoring.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index a4ede21..38b0eca 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -124,7 +124,7 @@ checkIfS3cmdInstalledAndConfigured() #check if s3cmd utility is configured checkIfS3cmdConfigured() { - var=$(s3cmd ls 2>/dev/null) + var=$(sudo s3cmd ls 2>/dev/null) if [ "$var" != "" ]; then if [ "$IS_S3CMD_CONFIGURED_BY_SCRIPT" == "false" ]; then logMsgToConfigSysLog "INFO" "INFO: s3cmd is already configured on your system" @@ -237,16 +237,17 @@ invokeS3FileMonitoring() fileNameWithExt=${f##*/} uniqueFileName=$(echo "$fileNameWithExt" | tr . _) var=$(file $f) - - if [ ${var##*\ } == "text" -o ${var##*\ } == "Text" ]; then - + + #it may be possible that the "text" may contain some uppercase letters like "Text" + var=$(echo $var | tr "[:upper:]" "[:lower:]") + + if [[ $var == *text* ]]; then LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$LOGGLY_S3_ALIAS LOGGLY_FILE_TO_MONITOR=$f constructFileVariables checkLogFileSize $LOGGLY_FILE_TO_MONITOR write21ConfFileContents IS_ANY_FILE_CONFIGURED="true" - else logMsgToConfigSysLog "WARN" "WARN: File $fileNameWithExt is not a text file. Ignoring." fi @@ -332,7 +333,7 @@ removeS3FileMonitoring() usage() { cat << EOF -usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3b s3bucketname ] [-s3l s3alias] +usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-s3url s3bucketname ] [-s3l s3alias] usage: configure-s3-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-s3l s3alias] usage: configure-s3-file-monitoring [-h for help] EOF @@ -364,7 +365,7 @@ while [ "$1" != "" ]; do -r | --rollback ) LOGGLY_ROLLBACK="true" ;; - -s3b | --s3bucketname ) shift + -s3url | --s3bucketname ) shift LOGGLY_S3_BUCKET_NAME=$1 echo "S3 Bucket Name: $LOGGLY_S3_BUCKET_NAME" ;; From 2f3cf7eda696abcb929c674b1deac1e440194b2c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 4 Aug 2014 18:18:45 +0530 Subject: [PATCH 63/95] Added more checks for the bucket name validation --- .../configure-s3-file-monitoring.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 38b0eca..5a9f5f8 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -171,18 +171,28 @@ configureS3cmd() #check if s3bucket is valid checkIfValidS3Bucket() { + #check if valid Bucket name + if [[ $LOGGLY_S3_BUCKET_NAME != s3://* ]]; then + logMsgToConfigSysLog "Error" "Error: Invalid s3 Bucket name. Bucket name should start with \"s3://\"" + exit 1 + fi + if [ "$LOGGLY_S3_BUCKET_NAME" != "" ]; then logMsgToConfigSysLog "INFO" "INFO: Check if valid S3 Bucket name." BUCKET_INFO=$(sudo s3cmd ls -r $LOGGLY_S3_BUCKET_NAME 2>&1) case $BUCKET_INFO in ERROR*) - logMsgToConfigSysLog "ERROR" "ERROR: Invalid S3 Bucket name $LOGGLY_S3_BUCKET_NAME." + #logging actual error message returned by s3cmd + logMsgToConfigSysLog "ERROR" "$BUCKET_INFO" exit 1 ;; "") logMsgToConfigSysLog "ERROR" "ERROR: No files found in the S3 Bucket $LOGGLY_S3_BUCKET_NAME." exit 1 ;; + *) + logMsgToConfigSysLog "INFO" "INFO: \"$LOGGLY_S3_BUCKET_NAME\" is a valid Bucket and accessible." + ;; esac fi } From b8abcd0999c642709025275a99b2c588ad7bc6da Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 5 Aug 2014 21:19:56 +0530 Subject: [PATCH 64/95] Removed extra quotes --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index bfa0a7f..30f5b34 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -281,7 +281,7 @@ checkIfLogglyServersAccessible() exit 1 fi - echo "INFO: Checking if \"$LOGGLY_ACCOUNT\" subdomain is valid." + echo "INFO: Checking if '$LOGGLY_ACCOUNT' subdomain is valid." if [ $(curl -s --head --request GET $LOGGLY_ACCOUNT_URL/login | grep "200 OK" | wc -l) == 1 ]; then echo "INFO: $LOGGLY_ACCOUNT_URL is valid and reachable." else From 7eac67694621692dc39d1c97cb411f320310a570 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 5 Aug 2014 22:21:45 +0530 Subject: [PATCH 65/95] Updated script to support s3 file monitoring --- .../configure-file-monitoring.sh | 123 +++--- .../configure-file-monitoring.sh | 359 ------------------ 2 files changed, 67 insertions(+), 415 deletions(-) delete mode 100644 Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 482a197..95cd14f 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.2 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -28,6 +28,9 @@ FILE_SYSLOG_CONFFILE_BACKUP= MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" +#this variable is set if the script is invoked via some other calling script +IS_FILE_MONITOR_SCRIPT_INVOKED="false" + ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -257,7 +260,9 @@ checkIfFileLogsMadeToLoggly() if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." - exit 0 + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then + exit 0 + fi fi } @@ -273,11 +278,15 @@ checkIfConfFileExist() #remove 21.conf file remove21ConfFile() { - echo "INFO: Deleting the loggly file syslog conf file." + echo "INFO: Deleting the loggly syslog conf file $FILE_SYSLOG_CONFFILE." if [ -f "$FILE_SYSLOG_CONFFILE" ]; then sudo rm -rf "$FILE_SYSLOG_CONFFILE" - fi - echo "INFO: Removed all the modified files." + if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then + echo "INFO: Removed all the modified files." + fi + else + logMsgToConfigSysLog "WARN" "WARN: $FILE_SYSLOG_CONFFILE file was not found." + fi } #display usage syntax @@ -291,58 +300,60 @@ EOF } ########## Get Inputs from User - Start ########## +if [ "$1" != "being-invoked" ]; then + if [ $# -eq 0 ]; then + usage + exit + else + while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --rollback ) + LOGGLY_ROLLBACK="true" + ;; + -f | --filename ) shift + #LOGGLY_FILE_TO_MONITOR=$1 + LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") + echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" + ;; + -l | --filealias ) shift + LOGGLY_FILE_TO_MONITOR_ALIAS=$1 + echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" + ;; + -h | --help) + usage + exit + ;; + esac + shift + done + fi -if [ $# -eq 0 ]; then - usage - exit -else -while [ "$1" != "" ]; do - case $1 in - -t | --token ) shift - LOGGLY_AUTH_TOKEN=$1 - echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" - ;; - -a | --account ) shift - LOGGLY_ACCOUNT=$1 - echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" - ;; - -u | --username ) shift - LOGGLY_USERNAME=$1 - echo "Username is set" - ;; - -p | --password ) shift - LOGGLY_PASSWORD=$1 - ;; - -r | --rollback ) - LOGGLY_ROLLBACK="true" - ;; - -f | --filename ) shift - #LOGGLY_FILE_TO_MONITOR=$1 - LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") - echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" - ;; - -l | --filealias ) shift - LOGGLY_FILE_TO_MONITOR_ALIAS=$1 - echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" - ;; - -h | --help) - usage - exit - ;; - esac - shift -done -fi - -if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_FILE_TO_MONITOR" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then - if [ "$LOGGLY_PASSWORD" = "" ]; then - getPassword + if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_FILE_TO_MONITOR" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConfForFile + elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then + removeLogglyConfForFile + else + usage fi - installLogglyConfForFile -elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then - removeLogglyConfForFile else - usage + IS_FILE_MONITOR_SCRIPT_INVOKED="true" fi - -########## Get Inputs from User - End ########## +########## Get Inputs from User - End ########## \ No newline at end of file diff --git a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh deleted file mode 100644 index 95cd14f..0000000 --- a/Modular Scripts/S3Logs Monitoring/configure-file-monitoring.sh +++ /dev/null @@ -1,359 +0,0 @@ -#!/bin/bash - -#downloads configure-linux.sh -echo "INFO: Downloading dependencies - configure-linux.sh" -curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh -source configure-linux.sh "being-invoked" - -########## Variable Declarations - Start ########## -#name of the current script -SCRIPT_NAME=configure-file-monitoring.sh -#version of the current script -SCRIPT_VERSION=1.2 - -#file to monitor (contains complete path and file name) provided by user -LOGGLY_FILE_TO_MONITOR= - -#alias name, will be used as tag & state file name etc. provided by user -LOGGLY_FILE_TO_MONITOR_ALIAS= - -#file alias provided by the user -APP_TAG="\"file-alias\":\"\"" - -#name and location of syslog file -FILE_SYSLOG_CONFFILE= - -#name and location of syslog backup file -FILE_SYSLOG_CONFFILE_BACKUP= - -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available at https://www.loggly.com/docs/file-monitoring/" - -#this variable is set if the script is invoked via some other calling script -IS_FILE_MONITOR_SCRIPT_INVOKED="false" - -########## Variable Declarations - End ########## - -# executing the script for loggly to install and configure syslog -installLogglyConfForFile() -{ - #log message indicating starting of Loggly configuration - logMsgToConfigSysLog "INFO" "INFO: Initiating configure Loggly for file monitoring." - - #check if the linux environment is compatible for Loggly - checkLinuxLogglyCompatibility - - #checks if the file name contain spaces, if yes, the exit - checkIfFileLocationContainSpaces - - #construct variables using filename and filealias - constructFileVariables - - #check if file to monitor exists - checkIfFileExist - - #check if the alias is already taken - checkIfFileAliasExist - - #configure loggly for Linux - installLogglyConf - - #create 21.conf file - write21ConfFileContents - - #restart rsyslog - restartRsyslog - - #check for the log file size - checkLogFileSize $LOGGLY_FILE_TO_MONITOR - - #verify if the file logs made it to loggly - checkIfFileLogsMadeToLoggly - - #log success message - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Successfully configured to send $LOGGLY_FILE_TO_MONITOR logs via Loggly." -} - -#executing script to remove loggly configuration for File -removeLogglyConfForFile() -{ - logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." - - #check if the user has root permission to run this script - checkIfUserHasRootPrivileges - - #check if the OS is supported by the script. If no, then exit - checkIfSupportedOS - - #construct variables using filename and filealias - constructFileVariables - - #checks if the conf file exists. if not, then exit. - checkIfConfFileExist - - #remove 21.conf file - remove21ConfFile - - #restart rsyslog - restartRsyslog - - #log success message - logMsgToConfigSysLog "INFO" "INFO: Rollback completed." -} - -checkIfFileLocationContainSpaces() -{ - case "$LOGGLY_FILE_TO_MONITOR" in - *\ * ) - logMsgToConfigSysLog "ERROR" "ERROR: File location cannot contain spaces." - exit 1;; - *) ;; - esac -} - -constructFileVariables() -{ - #conf file name - FILE_SYSLOG_CONFFILE="$RSYSLOG_ETCDIR_CONF/21-filemonitoring-$LOGGLY_FILE_TO_MONITOR_ALIAS.conf" - - #conf file backup name - FILE_SYSLOG_CONFFILE_BACKUP="$FILE_SYSLOG_CONFFILE.loggly.bk" - - #application tag - APP_TAG="\"file-alias\":\"$LOGGLY_FILE_TO_MONITOR_ALIAS\"" -} - -#checks if the file to be monitored exist -checkIfFileExist() -{ - if [ -f "$LOGGLY_FILE_TO_MONITOR" ]; then - logMsgToConfigSysLog "INFO" "INFO: File $LOGGLY_FILE_TO_MONITOR exists." - else - logMsgToConfigSysLog "ERROR" "ERROR: File $LOGGLY_FILE_TO_MONITOR does not exist. Kindly recheck." - exit 1 - fi -} - -#check if the file alias is already taken -checkIfFileAliasExist() -{ - if [ -f "$FILE_SYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "WARN" "WARN: This file alias is already taken. You must choose a unique file alias for each file." - while true; do - read -p "Would you like to overwrite the configuration for this file alias (yes/no)?" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $FILE_SYSLOG_CONFFILE to $FILE_SYSLOG_CONFFILE_BACKUP"; - sudo mv -f $FILE_SYSLOG_CONFFILE $FILE_SYSLOG_CONFFILE_BACKUP; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Not overwriting the existing configuration. Exiting" - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done - fi -} - -#check the size of the log file. If the size is greater than 100MB give a warning to the user. If the file size is 0 -#then exit -checkLogFileSize() -{ - monitorFileSize=$(wc -c "$1" | cut -f 1 -d ' ') - if [ $monitorFileSize -ge 102400000 ]; then - logMsgToConfigSysLog "INFO" "INFO: " - while true; do - read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Continuing with File Loggly configuration."; - break;; - [Nn]* ) - logMsgToConfigSysLog "INFO" "INFO: Current size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes. Discontinuing with File Loggly configuration." - exit 1 - break;; - * ) echo "Please answer yes or no.";; - esac - done - elif [ $monitorFileSize -eq 0 ]; then - logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from $LOGGLY_FILE_TO_MONITOR so there won't be any data sent to Loggly. You can generate some logs by writing to this file." - exit 1 - else - logMsgToConfigSysLog "INFO" "INFO: File size of $LOGGLY_FILE_TO_MONITOR is $monitorFileSize bytes." - fi -} - -#function to write the contents of syslog config file -write21ConfFileContents() -{ - logMsgToConfigSysLog "INFO" "INFO: Creating file $FILE_SYSLOG_CONFFILE" - sudo touch $FILE_SYSLOG_CONFFILE - sudo chmod o+w $FILE_SYSLOG_CONFFILE - - imfileStr="\$ModLoad imfile - \$InputFilePollInterval 10 - \$WorkDirectory $RSYSLOG_DIR - " - if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then - imfileStr+="\$PrivDropToGroup adm - " - fi - - imfileStr+=" - # File access file: - \$InputFileName $LOGGLY_FILE_TO_MONITOR - \$InputFileTag $LOGGLY_FILE_TO_MONITOR_ALIAS: - \$InputFileStateFile stat-$LOGGLY_FILE_TO_MONITOR_ALIAS - \$InputFileSeverity info - \$InputFilePersistStateInterval 20000 - \$InputRunFileMonitor - - #Add a tag for file events - \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"file\\\"] %msg%\n\" - - if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;LogglyFormatFile - if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ - " - - #write to 21-.conf file -sudo cat << EOIPFW >> $FILE_SYSLOG_CONFFILE -$imfileStr -EOIPFW - -} - -#checks if the apache logs made to loggly -checkIfFileLogsMadeToLoggly() -{ - counter=1 - maxCounter=10 - - fileInitialLogCount=0 - fileLatestLogCount=0 - queryParam="syslog.appName%3A$LOGGLY_FILE_TO_MONITOR_ALIAS&from=-15m&until=now&size=1" - - queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" - logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" - - logMsgToConfigSysLog "INFO" "INFO: Getting initial log count." - #get the initial count of file logs for past 15 minutes - searchAndFetch fileInitialLogCount "$queryUrl" - - logMsgToConfigSysLog "INFO" "INFO: Verifying if the logs made it to Loggly." - logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." - #get the final count of file logs for past 15 minutes - searchAndFetch fileLatestLogCount "$queryUrl" - let counter=$counter+1 - - while [ "$fileLatestLogCount" -le "$fileInitialLogCount" ]; do - echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." - sleep 30 - echo "INFO: Done waiting. Verifying again." - logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." - searchAndFetch fileLatestLogCount "$queryUrl" - let counter=$counter+1 - if [ "$counter" -gt "$maxCounter" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." - exit 1 - fi - done - - if [ "$fileLatestLogCount" -gt "$fileInitialLogCount" ]; then - logMsgToConfigSysLog "SUCCESS" "SUCCESS: Logs successfully transferred to Loggly! You are now sending $LOGGLY_FILE_TO_MONITOR logs to Loggly." - if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then - exit 0 - fi - fi -} - -#checks if the conf file exist. Name of conf file is constructed using the file alias name provided -checkIfConfFileExist() -{ - if [ ! -f "$FILE_SYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Invalid File Alias provided." - exit 1 - fi -} - -#remove 21.conf file -remove21ConfFile() -{ - echo "INFO: Deleting the loggly syslog conf file $FILE_SYSLOG_CONFFILE." - if [ -f "$FILE_SYSLOG_CONFFILE" ]; then - sudo rm -rf "$FILE_SYSLOG_CONFFILE" - if [ "$IS_FILE_MONITOR_SCRIPT_INVOKED" = "false" ]; then - echo "INFO: Removed all the modified files." - fi - else - logMsgToConfigSysLog "WARN" "WARN: $FILE_SYSLOG_CONFFILE file was not found." - fi -} - -#display usage syntax -usage() -{ -cat << EOF -usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-l filealias] -usage: configure-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] -usage: configure-file-monitoring [-h for help] -EOF -} - -########## Get Inputs from User - Start ########## -if [ "$1" != "being-invoked" ]; then - if [ $# -eq 0 ]; then - usage - exit - else - while [ "$1" != "" ]; do - case $1 in - -t | --token ) shift - LOGGLY_AUTH_TOKEN=$1 - echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" - ;; - -a | --account ) shift - LOGGLY_ACCOUNT=$1 - echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" - ;; - -u | --username ) shift - LOGGLY_USERNAME=$1 - echo "Username is set" - ;; - -p | --password ) shift - LOGGLY_PASSWORD=$1 - ;; - -r | --rollback ) - LOGGLY_ROLLBACK="true" - ;; - -f | --filename ) shift - #LOGGLY_FILE_TO_MONITOR=$1 - LOGGLY_FILE_TO_MONITOR=$(readlink -f "$1") - echo "File to monitor: $LOGGLY_FILE_TO_MONITOR" - ;; - -l | --filealias ) shift - LOGGLY_FILE_TO_MONITOR_ALIAS=$1 - echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" - ;; - -h | --help) - usage - exit - ;; - esac - shift - done - fi - - if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" -a "$LOGGLY_FILE_TO_MONITOR" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then - if [ "$LOGGLY_PASSWORD" = "" ]; then - getPassword - fi - installLogglyConfForFile - elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_FILE_TO_MONITOR_ALIAS" != "" ]; then - removeLogglyConfForFile - else - usage - fi -else - IS_FILE_MONITOR_SCRIPT_INVOKED="true" -fi -########## Get Inputs from User - End ########## \ No newline at end of file From a9436f9d0e564b8d6ebf763eefa65bcb7e0551fc Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 5 Aug 2014 22:29:24 +0530 Subject: [PATCH 66/95] Added check for the directory --- .../configure-s3-file-monitoring.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 5a9f5f8..38b880f 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -1,8 +1,7 @@ #!/bin/bash -#downloads configure-linux.sh -#echo "INFO: Downloading dependencies - configure-file-monitoring.sh" -#curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +echo "INFO: Downloading dependencies - configure-file-monitoring.sh" +curl -s -o configure-file-monitoring.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Modular%20Scripts/File%20Monitoring/configure-file-monitoring.sh source configure-file-monitoring.sh "being-invoked" ########## Variable Declarations - Start ########## @@ -76,7 +75,7 @@ installLogglyConfForS3() fi #delete temporary directory - deleteTempDir + #deleteTempDir } @@ -173,7 +172,7 @@ checkIfValidS3Bucket() { #check if valid Bucket name if [[ $LOGGLY_S3_BUCKET_NAME != s3://* ]]; then - logMsgToConfigSysLog "Error" "Error: Invalid s3 Bucket name. Bucket name should start with \"s3://\"" + logMsgToConfigSysLog "Error" "Error: Invalid s3 Bucket name. Bucket name should start with 's3://'" exit 1 fi @@ -191,7 +190,7 @@ checkIfValidS3Bucket() exit 1 ;; *) - logMsgToConfigSysLog "INFO" "INFO: \"$LOGGLY_S3_BUCKET_NAME\" is a valid Bucket and accessible." + logMsgToConfigSysLog "INFO" "INFO: '$LOGGLY_S3_BUCKET_NAME' is a valid Bucket and accessible." ;; esac fi @@ -258,7 +257,8 @@ invokeS3FileMonitoring() checkLogFileSize $LOGGLY_FILE_TO_MONITOR write21ConfFileContents IS_ANY_FILE_CONFIGURED="true" - else + #ignoring directory + elif [[ $var != *directory* ]]; then logMsgToConfigSysLog "WARN" "WARN: File $fileNameWithExt is not a text file. Ignoring." fi done From 991e9fb423945fb96464cc44523baed6516debc0 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 6 Aug 2014 20:21:27 +0530 Subject: [PATCH 67/95] Delete Temp Folder after Rollback --- .../S3Logs Monitoring/configure-s3-file-monitoring.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 38b880f..1075d82 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -96,6 +96,10 @@ removeLogglyConfForS3() #remove file monitoring removeS3FileMonitoring + #delete temporary directory if exists + TEMP_DIR=/tmp/$LOGGLY_S3_ALIAS + deleteTempDir + #log success message logMsgToConfigSysLog "INFO" "INFO: Rollback completed." } @@ -269,8 +273,10 @@ invokeS3FileMonitoring() } deleteTempDir() -{ - rm -fr $TEMP_DIR +{ + if [ -d "$TEMP_DIR" ]; then + sudo rm -fr $TEMP_DIR + fi } checkIfS3LogsMadeToLoggly() From c22bc30d0c6bfe50732fc9b1e4bec646eaac6618 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 7 Aug 2014 22:52:35 +0530 Subject: [PATCH 68/95] Added dynamic tag to modify from other scripts --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 95cd14f..b4039ad 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -31,6 +31,9 @@ MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available #this variable is set if the script is invoked via some other calling script IS_FILE_MONITOR_SCRIPT_INVOKED="false" +#file as tag sent with the logs +LOGGLY_FILE_TAG="file" + ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -209,7 +212,7 @@ write21ConfFileContents() \$InputRunFileMonitor #Add a tag for file events - \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"file\\\"] %msg%\n\" + \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;LogglyFormatFile if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ From 991912559b2e472e4346c7f02ab54c4c5f975731 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 7 Aug 2014 23:09:45 +0530 Subject: [PATCH 69/95] Fixed a bug --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b4039ad..9a7f79c 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -32,7 +32,7 @@ MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available IS_FILE_MONITOR_SCRIPT_INVOKED="false" #file as tag sent with the logs -LOGGLY_FILE_TAG="file" +LOGGLY_FILE_TAG= ########## Variable Declarations - End ########## @@ -49,6 +49,7 @@ installLogglyConfForFile() checkIfFileLocationContainSpaces #construct variables using filename and filealias + LOGGLY_FILE_TAG="file" constructFileVariables #check if file to monitor exists From cda8b4bd37c62b68ada745657a75a4588aec63c9 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 7 Aug 2014 23:34:01 +0530 Subject: [PATCH 70/95] Revert "Fixed a bug" This reverts commit 991912559b2e472e4346c7f02ab54c4c5f975731. --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 9a7f79c..b4039ad 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -32,7 +32,7 @@ MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure a file is available IS_FILE_MONITOR_SCRIPT_INVOKED="false" #file as tag sent with the logs -LOGGLY_FILE_TAG= +LOGGLY_FILE_TAG="file" ########## Variable Declarations - End ########## @@ -49,7 +49,6 @@ installLogglyConfForFile() checkIfFileLocationContainSpaces #construct variables using filename and filealias - LOGGLY_FILE_TAG="file" constructFileVariables #check if file to monitor exists From c7d1e723814e84fa671681fd38796a20b7c13d2e Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 8 Aug 2014 00:01:25 +0530 Subject: [PATCH 71/95] S3 Script: Added "s3file" tag --- .../S3Logs Monitoring/configure-s3-file-monitoring.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 1075d82..a9a37c3 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -257,6 +257,7 @@ invokeS3FileMonitoring() if [[ $var == *text* ]]; then LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$LOGGLY_S3_ALIAS LOGGLY_FILE_TO_MONITOR=$f + LOGGLY_FILE_TAG="s3file" constructFileVariables checkLogFileSize $LOGGLY_FILE_TO_MONITOR write21ConfFileContents From 7dbf18aa663d19bfa1092524a35089f41572545a Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Mon, 18 Aug 2014 15:48:08 +0530 Subject: [PATCH 72/95] Updated script to support multiple Linux machines --- Modular Scripts/Apache2/configure-apache.sh | 27 +++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index d4cb448..48bc27f 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-apache.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.2 #we have not found the apache version yet at this point in the script APP_TAG="\"apache-version\":\"\"" @@ -122,23 +122,20 @@ checkApacheDetails() #Get the apache service name on various linux flavors getApacheServiceName() { - case "$LINUX_DIST" in - *"Ubuntu"* ) - SERVICE="apache2" - APACHE_ACCESS_LOG_FILE="access.log" - APACHE_ERROR_LOG_FILE="error.log" - ;; - *"RedHat"* ) - SERVICE="httpd" - APACHE_ACCESS_LOG_FILE="access_log" - APACHE_ERROR_LOG_FILE="error_log" - ;; - *"CentOS"* ) + #checking if the Linux is yum based or apt-get based + YUM_BASED=$(command -v yum) + APT_GET_BASED=$(command -v apt-get) + + if [ "$YUM_BASED" != "" ]; then SERVICE="httpd" APACHE_ACCESS_LOG_FILE="access_log" APACHE_ERROR_LOG_FILE="error_log" - ;; - esac + + elif [ "$APT_GET_BASED" != "" ]; then + SERVICE="apache2" + APACHE_ACCESS_LOG_FILE="access.log" + APACHE_ERROR_LOG_FILE="error.log" + fi } #sets apache variables which will be used across various functions From b84686d7bf9caba38a4b7037a243805b238d3ded Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 19 Aug 2014 17:56:57 +0530 Subject: [PATCH 73/95] File-Monitoring- --- .../File Monitoring/configure-file-monitoring.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b4039ad..b019edc 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.2 +SCRIPT_VERSION=1.3 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -34,6 +34,9 @@ IS_FILE_MONITOR_SCRIPT_INVOKED="false" #file as tag sent with the logs LOGGLY_FILE_TAG="file" +#format name for the conf file. Can be set by calling script +CONF_FILE_FORMAT_NAME="LogglyFormatFile" + ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -212,9 +215,9 @@ write21ConfFileContents() \$InputRunFileMonitor #Add a tag for file events - \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" + \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" - if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;LogglyFormatFile + if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;$CONF_FILE_FORMAT_NAME if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ " From b7f74437cdf2684662eaea689a5363eb8aff8619 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 19 Aug 2014 17:57:37 +0530 Subject: [PATCH 74/95] Revert "File-Monitoring-" This reverts commit b84686d7bf9caba38a4b7037a243805b238d3ded. --- .../File Monitoring/configure-file-monitoring.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b019edc..b4039ad 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.3 +SCRIPT_VERSION=1.2 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -34,9 +34,6 @@ IS_FILE_MONITOR_SCRIPT_INVOKED="false" #file as tag sent with the logs LOGGLY_FILE_TAG="file" -#format name for the conf file. Can be set by calling script -CONF_FILE_FORMAT_NAME="LogglyFormatFile" - ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -215,9 +212,9 @@ write21ConfFileContents() \$InputRunFileMonitor #Add a tag for file events - \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" + \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" - if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;$CONF_FILE_FORMAT_NAME + if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;LogglyFormatFile if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ " From a2d377e2f2f46416fc4d98de2102ae7a53ebe86e Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 19 Aug 2014 18:01:36 +0530 Subject: [PATCH 75/95] File-monitoring: Added support to set Format name Conf Format name can be set from the calling script to override tag value --- .../File Monitoring/configure-file-monitoring.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b4039ad..b019edc 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.2 +SCRIPT_VERSION=1.3 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -34,6 +34,9 @@ IS_FILE_MONITOR_SCRIPT_INVOKED="false" #file as tag sent with the logs LOGGLY_FILE_TAG="file" +#format name for the conf file. Can be set by calling script +CONF_FILE_FORMAT_NAME="LogglyFormatFile" + ########## Variable Declarations - End ########## # executing the script for loggly to install and configure syslog @@ -212,9 +215,9 @@ write21ConfFileContents() \$InputRunFileMonitor #Add a tag for file events - \$template LogglyFormatFile,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" + \$template $CONF_FILE_FORMAT_NAME,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"$LOGGLY_FILE_TAG\\\"] %msg%\n\" - if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;LogglyFormatFile + if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then @@logs-01.loggly.com:514;$CONF_FILE_FORMAT_NAME if \$programname == '$LOGGLY_FILE_TO_MONITOR_ALIAS' then ~ " From 408af5556b1e1d4655e8c7088ac9725303798a23 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 19 Aug 2014 18:02:06 +0530 Subject: [PATCH 76/95] S3 Script: Setting New Format name --- .../S3Logs Monitoring/configure-s3-file-monitoring.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index a9a37c3..986f5a2 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -8,7 +8,7 @@ source configure-file-monitoring.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-s3-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.0 +SCRIPT_VERSION=1.1 #s3 bucket name to configure LOGGLY_S3_BUCKET_NAME= @@ -258,6 +258,7 @@ invokeS3FileMonitoring() LOGGLY_FILE_TO_MONITOR_ALIAS=$uniqueFileName-$LOGGLY_S3_ALIAS LOGGLY_FILE_TO_MONITOR=$f LOGGLY_FILE_TAG="s3file" + CONF_FILE_FORMAT_NAME="LogglyFormatS3" constructFileVariables checkLogFileSize $LOGGLY_FILE_TO_MONITOR write21ConfFileContents From 20e0ef36434774f76ba98fbbffb396e609c59055 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 21 Aug 2014 22:50:54 +0530 Subject: [PATCH 77/95] Updated Path for configure-file-monitoring.sh --- Modular Scripts/Apache2/configure-apache.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index 48bc27f..cc454fd 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -2,7 +2,7 @@ #downloads configure-linux.sh echo "INFO: Downloading dependencies - configure-linux.sh" -curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +curl -s -o configure-linux.sh https://www.loggly.com/install/configure-linux.sh source configure-linux.sh "being-invoked" ########## Variable Declarations - Start ########## @@ -375,4 +375,4 @@ else usage fi -########## Get Inputs from User - End ########## \ No newline at end of file +########## Get Inputs from User - End ########## From 8989253cd988f5750558f8f1d942cc716dec0460 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 21 Aug 2014 22:51:51 +0530 Subject: [PATCH 78/95] Updated Path for configure-linux.sh --- Modular Scripts/File Monitoring/configure-file-monitoring.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index b019edc..5cee872 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -2,7 +2,7 @@ #downloads configure-linux.sh echo "INFO: Downloading dependencies - configure-linux.sh" -curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +curl -s -o configure-linux.sh https://www.loggly.com/install/configure-linux.sh source configure-linux.sh "being-invoked" ########## Variable Declarations - Start ########## @@ -362,4 +362,4 @@ if [ "$1" != "being-invoked" ]; then else IS_FILE_MONITOR_SCRIPT_INVOKED="true" fi -########## Get Inputs from User - End ########## \ No newline at end of file +########## Get Inputs from User - End ########## From 8b62c0fd03d136db01fe731ff0f016bc7a38e0b6 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 21 Aug 2014 22:52:29 +0530 Subject: [PATCH 79/95] Updated Path for configure-file-monitoring.sh --- .../S3Logs Monitoring/configure-s3-file-monitoring.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index 986f5a2..cdba46c 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -1,7 +1,7 @@ #!/bin/bash echo "INFO: Downloading dependencies - configure-file-monitoring.sh" -curl -s -o configure-file-monitoring.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Modular%20Scripts/File%20Monitoring/configure-file-monitoring.sh +curl -s -o configure-file-monitoring.sh https://www.loggly.com/install/configure-file-monitoring.sh source configure-file-monitoring.sh "being-invoked" ########## Variable Declarations - Start ########## From c716576a9ce328b398c8440f9e0bb618a85b9a70 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Thu, 21 Aug 2014 22:53:05 +0530 Subject: [PATCH 80/95] Updated Path for configure-linux.sh --- Modular Scripts/Tomcat/configure-tomcat.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index af2a1b3..04b16c8 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -2,7 +2,7 @@ #downloads configure-linux.sh echo "INFO: Downloading dependencies - configure-linux.sh" -curl -s -o configure-linux.sh https://raw.githubusercontent.com/psquickitjayant/install-script/master/Linux%20Script/configure-linux.sh +curl -s -o configure-linux.sh https://www.loggly.com/install/configure-linux.sh source configure-linux.sh "being-invoked" ########## Variable Declarations - Start ########## From fd0d765c51f16be866ed6668332b412e8945c3a2 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 17:06:14 +0530 Subject: [PATCH 81/95] Merged script-dev changes Also updated dependencies link to Loggly --- Linux Script/configure-linux.sh | 76 +++++++++---- Modular Scripts/Apache2/configure-apache.sh | 2 +- .../configure-file-monitoring.sh | 19 +++- .../configure-s3-file-monitoring.sh | 102 +++++++++++++++--- Modular Scripts/Tomcat/configure-tomcat.sh | 2 +- 5 files changed, 162 insertions(+), 39 deletions(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index 30f5b34..db1bbac 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -15,7 +15,7 @@ function ctrl_c() { #name of the current script. This will get overwritten by the child script which calls this SCRIPT_NAME=configure-linux.sh #version of the current script. This will get overwritten by the child script which calls this -SCRIPT_VERSION=1.5 +SCRIPT_VERSION=1.7 #application tag. This will get overwritten by the child script which calls this APP_TAG= @@ -145,10 +145,12 @@ installLogglyConf() if [ "$LINUX_DO_VERIFICATION" = "true" ]; then #check if the logs are going to loggly fro linux system now checkIfLogsMadeToLoggly + fi - #log success message + if [ "$IS_INVOKED" = "" ]; then logMsgToConfigSysLog "SUCCESS" "SUCCESS: Linux system successfully configured to send logs via Loggly." fi + } #remove loggly configuration from Linux system @@ -269,7 +271,7 @@ checkIfLogglyServersAccessible() if [ $(ping -c 1 $LOGS_01_HOST | grep "1 packets transmitted, 1 received, 0% packet loss" | wc -l) == 1 ]; then echo "INFO: $LOGS_01_HOST is reachable." else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings. Continuing to configure Loggly on your system." + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable. Please check your network and firewall settings." exit 1 fi @@ -277,7 +279,7 @@ checkIfLogglyServersAccessible() if [ $(curl --connect-timeout 10 $LOGS_01_HOST:$LOGGLY_SYSLOG_PORT 2>&1 | grep "Empty reply from server" | wc -l) == 1 ]; then echo "INFO: $LOGS_01_HOST is reachable via $LOGGLY_SYSLOG_PORT port." else - logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings. Continuing to configure Loggly on your system." + logMsgToConfigSysLog "ERROR" "ERROR: $LOGS_01_HOST is not reachable via $LOGGLY_SYSLOG_PORT port. Please check your network and firewall settings." exit 1 fi @@ -401,22 +403,8 @@ write22LogglyConfFile() { echo "INFO: Checking if loggly sysconf file $LOGGLY_RSYSLOG_CONFFILE exist." if [ -f "$LOGGLY_RSYSLOG_CONFFILE" ]; then - logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." - while true; do - read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn - case $yn in - [Yy]* ) - logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; - sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; - checkAuthTokenAndWriteContents; - break;; - [Nn]* ) - LINUX_DO_VERIFICATION="false" - logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." - break;; - * ) echo "Please answer yes or no.";; - esac - done + logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE already exist." + checkIfConfigurationChanged else logMsgToConfigSysLog "INFO" "INFO: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE does not exist, creating file $LOGGLY_RSYSLOG_CONFFILE" checkAuthTokenAndWriteContents @@ -435,6 +423,48 @@ checkAuthTokenAndWriteContents() fi } +#matches if the content of 22-loggly.conf content is changed +checkIfConfigurationChanged() +{ + ASK_FOR_VERIFICATION="false" + + #strings to be checked which should be present in the existing 22-loggly.conf. + #If these strings are not same then a warning message will be shown to user to update the 22-loggly.conf file + STR_TO_BE_CHECKED[0]="\$template LogglyFormat,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@$LOGGLY_DISTRIBUTION_ID] %msg%\"" + STR_TO_BE_CHECKED[1]="*.* @@$LOGS_01_HOST:$LOGGLY_SYSLOG_PORT;LogglyFormat" + + for i in "${STR_TO_BE_CHECKED[@]}" + do + if ! sudo grep -Fxq "$i" $LOGGLY_RSYSLOG_CONFFILE; then + ASK_FOR_VERIFICATION="true" + break; + fi + done + + if [ "$ASK_FOR_VERIFICATION" == "true" ]; then + logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE content is incorrect." + while true; + do + read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $LOGGLY_RSYSLOG_CONFFILE to $LOGGLY_RSYSLOG_CONFFILE_BACKUP"; + sudo mv -f $LOGGLY_RSYSLOG_CONFFILE $LOGGLY_RSYSLOG_CONFFILE_BACKUP; + checkAuthTokenAndWriteContents; + break;; + [Nn]* ) + LINUX_DO_VERIFICATION="false" + logMsgToConfigSysLog "INFO" "INFO: Skipping Linux verification." + break;; + * ) echo "Please answer yes or no.";; + esac + done + else + LINUX_DO_VERIFICATION="false" + fi + +} + #write the contents to 22-loggly.conf file writeContents() { @@ -506,7 +536,7 @@ checkIfLogsMadeToLoggly() let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then MANUAL_CONFIG_INSTRUCTION=$MANUAL_CONFIG_INSTRUCTION" Rsyslog troubleshooting instructions are available at https://www.loggly.com/docs/troubleshooting-rsyslog/" - logMsgToConfigSysLog "ERROR" "ERROR: Verification logs did not make it to Loggly in time. Please check your token & network/firewall settings and retry." + logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi done @@ -601,9 +631,9 @@ logMsgToConfigSysLog() sendPayloadToConfigSysLog() { if [ "$APP_TAG" = "" ]; then - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\"}" + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", \"messages\":\"$2\",\"rsyslog-version\":\"$RSYSLOG_VERSION\"}" else - var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\"}" + var="{\"sub-domain\":\"$LOGGLY_ACCOUNT\", \"user-name\":\"$LOGGLY_USERNAME\", \"customer-token\":\"$LOGGLY_AUTH_TOKEN\", \"host-name\":\"$HOST_NAME\", \"script-name\":\"$SCRIPT_NAME\", \"script-version\":\"$SCRIPT_VERSION\", \"status\":\"$1\", \"time-stamp\":\"$currentTime\", \"linux-distribution\":\"$LINUX_DIST\", $APP_TAG, \"messages\":\"$2\",\"rsyslog-version\":\"$RSYSLOG_VERSION\"}" fi curl -s -H "content-type:application/json" -d "$var" $LOGS_01_URL/inputs/$3 > /dev/null 2>&1 } diff --git a/Modular Scripts/Apache2/configure-apache.sh b/Modular Scripts/Apache2/configure-apache.sh index cc454fd..e1ccf12 100644 --- a/Modular Scripts/Apache2/configure-apache.sh +++ b/Modular Scripts/Apache2/configure-apache.sh @@ -297,7 +297,7 @@ checkIfApacheLogsMadeToLoggly() searchAndFetch apacheLatestLogCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + logMsgToConfigSysLog "ERROR" "ERROR: Apache logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi done diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 5cee872..69bb240 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.3 +SCRIPT_VERSION=1.4 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -57,6 +57,9 @@ installLogglyConfForFile() #check if file to monitor exists checkIfFileExist + #checks if the file has proper read permission + checkFileReadPermission + #check if the alias is already taken checkIfFileAliasExist @@ -189,6 +192,18 @@ checkLogFileSize() fi } + +#checks the input file has proper read permissions +checkFileReadPermission() +{ + FILE_PERMISSIONS=$(ls -l $LOGGLY_FILE_TO_MONITOR) + #checking if the file has read permission for others + PERMISSION_READ_OTHERS=${FILE_PERMISSIONS:7:1} + if [ $PERMISSION_READ_OTHERS != r ]; then + logMsgToConfigSysLog "WARN" "WARN: $LOGGLY_FILE_TO_MONITOR does not have proper read permissions. Verification step may fail." + fi +} + #function to write the contents of syslog config file write21ConfFileContents() { @@ -259,7 +274,7 @@ checkIfFileLogsMadeToLoggly() searchAndFetch fileLatestLogCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + logMsgToConfigSysLog "ERROR" "ERROR: File logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi done diff --git a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh index cdba46c..f14d841 100644 --- a/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh +++ b/Modular Scripts/S3Logs Monitoring/configure-s3-file-monitoring.sh @@ -8,7 +8,7 @@ source configure-file-monitoring.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-s3-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.1 +SCRIPT_VERSION=1.3 #s3 bucket name to configure LOGGLY_S3_BUCKET_NAME= @@ -67,15 +67,16 @@ installLogglyConfForS3() #invoke file monitoring on each file after checking if it is a text file or not invokeS3FileMonitoring + #install a cron job to sync the downloaded files after each 5 minutes + installCronToSyncS3BucketPeriodically + if [ "$IS_ANY_FILE_CONFIGURED" != "false" ]; then #check if s3 logs made it to loggly checkIfS3LogsMadeToLoggly else logMsgToConfigSysLog "WARN" "WARN: Did not find any files to configure. Nothing to do." fi - - #delete temporary directory - #deleteTempDir + } @@ -95,9 +96,12 @@ removeLogglyConfForS3() #remove file monitoring removeS3FileMonitoring + + #delete the s3 sync cron from the crontab + deleteS3CronFromCrontab #delete temporary directory if exists - TEMP_DIR=/tmp/$LOGGLY_S3_ALIAS + TEMP_DIR=/tmp/s3monitoring/$LOGGLY_S3_ALIAS deleteTempDir #log success message @@ -165,8 +169,8 @@ downloadS3cmd() #configure s3cmd configureS3cmd() { - s3cmd --configure - IS_S3CMD_CONFIGURED_BY_SCRIPT="true" + sudo s3cmd --configure + IS_S3CMD_CONFIGURED_BY_SCRIPT="true" #check if s3cmd configured successfully now checkIfS3cmdConfigured } @@ -203,7 +207,7 @@ checkIfValidS3Bucket() createTempDir() { - TEMP_DIR=/tmp/$LOGGLY_S3_ALIAS + TEMP_DIR=/tmp/s3monitoring/$LOGGLY_S3_ALIAS if [ -d "$TEMP_DIR" ]; then if [ "$(ls -A $TEMP_DIR)" ]; then logMsgToConfigSysLog "WARN" "WARN: There are some files/folders already present in $TEMP_DIR. If you continue, the files currently inside the $TEMP_DIR will also be configured to send logs to loggly." @@ -221,7 +225,12 @@ createTempDir() done fi else - mkdir /tmp/$LOGGLY_S3_ALIAS + if [ -d "/tmp/s3monitoring" ]; then + mkdir /tmp/s3monitoring/$LOGGLY_S3_ALIAS + else + mkdir /tmp/s3monitoring + mkdir /tmp/s3monitoring/$LOGGLY_S3_ALIAS + fi fi } @@ -231,7 +240,7 @@ downloadS3Bucket() #Files are downloaded in nested directory cd $TEMP_DIR echo "Downloading files, may take some time..." - s3cmd get -r -f $LOGGLY_S3_BUCKET_NAME > /dev/null 2>&1 + sudo s3cmd get -r -f $LOGGLY_S3_BUCKET_NAME > /dev/null 2>&1 if [ $? -ne 0 ]; then logMsgToConfigSysLog "ERROR" "ERROR: Error downloading files recursively from $LOGGLY_S3_BUCKET_NAME" exit 1 @@ -242,7 +251,7 @@ downloadS3Bucket() invokeS3FileMonitoring() { - dir=/tmp/$LOGGLY_S3_ALIAS + dir=/tmp/s3monitoring/$LOGGLY_S3_ALIAS #TODO: Not supporting multiple files with same name in different directories #only supporting file with naming convention *.* for f in $(find $dir -name '*') @@ -260,6 +269,7 @@ invokeS3FileMonitoring() LOGGLY_FILE_TAG="s3file" CONF_FILE_FORMAT_NAME="LogglyFormatS3" constructFileVariables + checkFileReadPermission checkLogFileSize $LOGGLY_FILE_TO_MONITOR write21ConfFileContents IS_ANY_FILE_CONFIGURED="true" @@ -274,6 +284,68 @@ invokeS3FileMonitoring() fi } +installCronToSyncS3BucketPeriodically() +{ + while true; do + read -p "Would you like install a Cron job to sync the files currently in your bucket every 5 minutes? (yes/no)" yn + case $yn in + [Yy]* ) + + #copying .s3cfg file to /root so that it can be used by crontab for sync + if ! sudo test -f "/root/.s3cfg" ; then + sudo cp $HOME/.s3cfg /root + fi + + CRON_FILE="/tmp/s3monitoring/cron_$LOGGLY_S3_ALIAS" + CRON_SYNC_PATH="/tmp/s3monitoring/$LOGGLY_S3_ALIAS" + + #checking if the provided s3 path if of directory or file + IS_DIR="true" + BUCKET_URL_LAST_VALUE=$(echo ${LOGGLY_S3_BUCKET_NAME##*/}) + + if [ "$BUCKET_URL_LAST_VALUE" != "" ]; then + for fle in $(find $CRON_SYNC_PATH -name $BUCKET_URL_LAST_VALUE) + do + if [ -f $fle ]; then + IS_DIR="false" + break + fi + done + fi + + #adding file name to the sync folder as the bucket path is + #provided upto a file + if [ "$IS_DIR" == "false" ]; then + CRON_SYNC_PATH="$CRON_SYNC_PATH/$BUCKET_URL_LAST_VALUE" + fi + + logMsgToConfigSysLog "INFO" "INFO: Creating a Cron job to sync $LOGGLY_S3_BUCKET_NAME files to $CRON_SYNC_PATH in every five minutes." + + #setting up cron job + CRON_JOB_TO_SYNC_S3_BUCKET="*/5 * * * * s3cmd sync $LOGGLY_S3_BUCKET_NAME --preserve $CRON_SYNC_PATH" + + EXISTING_CRONS=$(sudo crontab -l 2>&1) + case $EXISTING_CRONS in + no*) + ;; + *) + echo "$EXISTING_CRONS" >> $CRON_FILE + ;; + esac + + echo "$CRON_JOB_TO_SYNC_S3_BUCKET" >> $CRON_FILE + sudo crontab $CRON_FILE + sudo rm -fr $CRON_FILE + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Skipping Cron installation." + break;; + * ) echo "Please answer yes or no.";; + esac + done + +} + deleteTempDir() { if [ -d "$TEMP_DIR" ]; then @@ -307,7 +379,7 @@ checkIfS3LogsMadeToLoggly() searchAndFetch fileLatestLogCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + logMsgToConfigSysLog "ERROR" "ERROR: S3 logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi done @@ -347,6 +419,12 @@ removeS3FileMonitoring() restartRsyslog } +deleteS3CronFromCrontab() +{ + logMsgToConfigSysLog "INFO" "INFO: Deleting sync Cron." + sudo crontab -l | grep -v "/tmp/s3monitoring/$LOGGLY_S3_ALIAS" | crontab - +} + #display usage syntax usage() { diff --git a/Modular Scripts/Tomcat/configure-tomcat.sh b/Modular Scripts/Tomcat/configure-tomcat.sh index 04b16c8..91f92ea 100644 --- a/Modular Scripts/Tomcat/configure-tomcat.sh +++ b/Modular Scripts/Tomcat/configure-tomcat.sh @@ -568,7 +568,7 @@ checkIfTomcatLogsMadeToLoggly() searchAndFetch tomcatLatestLogCount "$queryUrl" let counter=$counter+1 if [ "$counter" -gt "$maxCounter" ]; then - logMsgToConfigSysLog "ERROR" "ERROR: Tomcat logs did not make to Loggly in time. Please check your token & network/firewall settings and retry." + logMsgToConfigSysLog "ERROR" "ERROR: Tomcat logs did not make to Loggly in time. Please check network and firewall settings and retry." exit 1 fi done From 4e389370a49764de8708a4664ba6febc942f116c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 17:24:13 +0530 Subject: [PATCH 82/95] Create README.md --- Modular Scripts/S3Logs Monitoring/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Modular Scripts/S3Logs Monitoring/README.md diff --git a/Modular Scripts/S3Logs Monitoring/README.md b/Modular Scripts/S3Logs Monitoring/README.md new file mode 100644 index 0000000..d144dfe --- /dev/null +++ b/Modular Scripts/S3Logs Monitoring/README.md @@ -0,0 +1,14 @@ +AWS S3 File Monitoring Script +====================== + +Configure your S3 bucket and file logs to send to Loggly with synchronization + + sudo bash configure-s3-file-monitoring.sh -a SUBDOMAIN -u USERNAME -s3url S3-BUCKET-PATH -s3l S3-BUCKET-ALIAS + +**Note:** S3 Bucket Alias should be unique for each file. + + + +Stop sending your S3 bucket logs to Loggly + + sudo ./configure-s3-file-monitoring.sh -a SUBDOMAIN -s3l S3-BUCKET-ALIAS -r From 1118c5bf5f4c419d5af173d021a9e6d227d0c219 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 17:25:12 +0530 Subject: [PATCH 83/95] Update README.md --- Modular Scripts/S3Logs Monitoring/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/S3Logs Monitoring/README.md b/Modular Scripts/S3Logs Monitoring/README.md index d144dfe..951405c 100644 --- a/Modular Scripts/S3Logs Monitoring/README.md +++ b/Modular Scripts/S3Logs Monitoring/README.md @@ -5,7 +5,7 @@ Configure your S3 bucket and file logs to send to Loggly with synchronization sudo bash configure-s3-file-monitoring.sh -a SUBDOMAIN -u USERNAME -s3url S3-BUCKET-PATH -s3l S3-BUCKET-ALIAS -**Note:** S3 Bucket Alias should be unique for each file. +**Note:** S3 Bucket Alias should be unique for each bucket. From a05c43ae73e960a132ef59f173a07a6b25f5eb0f Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 17:34:51 +0530 Subject: [PATCH 84/95] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 96b84fa..264bd41 100644 --- a/README.md +++ b/README.md @@ -13,3 +13,4 @@ You can view the **READMEs** of the various installation scripts at the followin 2. Apache Installation Script 3. File Monitoring Configuration Script 4. Tomcat Configuration Script +5. S3 Bucket Configuration Script From b896a25c205af41f80f5691cdeb71f2ec01593de Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 17:35:40 +0530 Subject: [PATCH 85/95] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 264bd41..62513af 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ install-script ============== -Loggly Sender Installation Script. Sends Linux, Tomcat, Apache and Text file logs/ contents to Loggly. +Loggly Sender Installation Script. Sends Linux, Tomcat, Apache, Text file and S3 bucket logs to Loggly. ***Note:*** These scripts support only Loggly **Gen2** account. From d17eca2f4427752255808de71cfa850eba5b8122 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 17:36:46 +0530 Subject: [PATCH 86/95] Update README.md --- Modular Scripts/S3Logs Monitoring/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/S3Logs Monitoring/README.md b/Modular Scripts/S3Logs Monitoring/README.md index 951405c..d667c16 100644 --- a/Modular Scripts/S3Logs Monitoring/README.md +++ b/Modular Scripts/S3Logs Monitoring/README.md @@ -11,4 +11,4 @@ Configure your S3 bucket and file logs to send to Loggly with synchronization Stop sending your S3 bucket logs to Loggly - sudo ./configure-s3-file-monitoring.sh -a SUBDOMAIN -s3l S3-BUCKET-ALIAS -r + sudo bash configure-s3-file-monitoring.sh -a SUBDOMAIN -s3l S3-BUCKET-ALIAS -r From 7e7bc4c2a9d77116d3b61e92e37da6336815d439 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 17:44:26 +0530 Subject: [PATCH 87/95] Updated ReadMes --- Linux Script/README.md | 5 ++--- Modular Scripts/Apache2/README.md | 5 ++--- Modular Scripts/File Monitoring/README.md | 5 ++--- Modular Scripts/Tomcat/README.md | 5 ++--- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/Linux Script/README.md b/Linux Script/README.md index 68b91fd..62f8faa 100644 --- a/Linux Script/README.md +++ b/Linux Script/README.md @@ -3,10 +3,9 @@ Linux Script Configure your Linux system to send syslogs to Loggly using the following command - chmod 755 configure-linux.sh - sudo ./configure-linux.sh -a SUBDOMAIN -u USERNAME + sudo bash configure-linux.sh -a SUBDOMAIN -u USERNAME Stop sending your Linux System logs to Loggly - sudo ./configure-linux.sh -a SUBDOMAIN -r + sudo bash configure-linux.sh -a SUBDOMAIN -r diff --git a/Modular Scripts/Apache2/README.md b/Modular Scripts/Apache2/README.md index 621f31e..1b7f393 100644 --- a/Modular Scripts/Apache2/README.md +++ b/Modular Scripts/Apache2/README.md @@ -3,9 +3,8 @@ Apache Script Configure your Apache server to send logs from access file and error file to Loggly - chmod 755 configure-apache.sh - sudo ./configure-apache -a SUBDOMAIN -u USERNAME + sudo bash configure-apache -a SUBDOMAIN -u USERNAME Stop sending your Apache logs to Loggly - sudo ./configure-apache.sh -a SUBDOMAIN -r + sudo bash configure-apache.sh -a SUBDOMAIN -r diff --git a/Modular Scripts/File Monitoring/README.md b/Modular Scripts/File Monitoring/README.md index 21d1797..79cbfbf 100644 --- a/Modular Scripts/File Monitoring/README.md +++ b/Modular Scripts/File Monitoring/README.md @@ -3,8 +3,7 @@ File Monitoring Script Configure your any text file to send it contents to Loggly - chmod 755 configure-file-monitoring.sh - sudo ./configure-file-monitoring.sh -a SUBDOMAIN -u USERNAME -f FILENAME -l FILE_ALIAS + sudo bash configure-file-monitoring.sh -a SUBDOMAIN -u USERNAME -f FILENAME -l FILE_ALIAS **Note:** File Alias should be unique for each file. @@ -12,4 +11,4 @@ Configure your any text file to send it contents to Loggly Stop sending your file contents to Loggly - sudo ./configure-file-monitoring.sh -a SUBDOMAIN -l FILE_ALIAS -r + sudo bash configure-file-monitoring.sh -a SUBDOMAIN -l FILE_ALIAS -r diff --git a/Modular Scripts/Tomcat/README.md b/Modular Scripts/Tomcat/README.md index e06c5e7..342ad1a 100644 --- a/Modular Scripts/Tomcat/README.md +++ b/Modular Scripts/Tomcat/README.md @@ -3,9 +3,8 @@ Tomcat Script Send your Tomcat logs to Loggly - chmod 755 configure-tomcat.sh - sudo ./configure-tomcat.sh -a SUBDOMAIN -u USERNAME + sudo bash configure-tomcat.sh -a SUBDOMAIN -u USERNAME Stop sending your Tomcat logs to Loggly - sudo ./configure-tomcat.sh -a SUBDOMAIN -r + sudo bash configure-tomcat.sh -a SUBDOMAIN -r From b0d97321509adabfd323102b61cab642d1724bb2 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 26 Sep 2014 19:07:43 +0530 Subject: [PATCH 88/95] Skipping permissions check on RedHat and CentOS --- .../configure-file-monitoring.sh | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index 69bb240..aa0f470 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -196,12 +196,25 @@ checkLogFileSize() #checks the input file has proper read permissions checkFileReadPermission() { - FILE_PERMISSIONS=$(ls -l $LOGGLY_FILE_TO_MONITOR) - #checking if the file has read permission for others - PERMISSION_READ_OTHERS=${FILE_PERMISSIONS:7:1} - if [ $PERMISSION_READ_OTHERS != r ]; then - logMsgToConfigSysLog "WARN" "WARN: $LOGGLY_FILE_TO_MONITOR does not have proper read permissions. Verification step may fail." - fi + + LINUX_DIST_IN_LOWER_CASE=$(echo $LINUX_DIST | tr "[:upper:]" "[:lower:]") + + #no need to check read permissions with RedHat and CentOS as they also work with ---------- (000)permissions + case "$LINUX_DIST_IN_LOWER_CASE" in + *"redhat"* ) + ;; + *"centos"* ) + ;; + * ) + FILE_PERMISSIONS=$(ls -l $LOGGLY_FILE_TO_MONITOR) + #checking if the file has read permission for others + PERMISSION_READ_OTHERS=${FILE_PERMISSIONS:7:1} + if [ $PERMISSION_READ_OTHERS != r ]; then + logMsgToConfigSysLog "WARN" "WARN: $LOGGLY_FILE_TO_MONITOR does not have proper read permissions. Verification step may fail." + fi + ;; + esac + } #function to write the contents of syslog config file From b55c6802e8001c1be05b09812da37732043b6dc9 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 30 Sep 2014 19:07:37 +0530 Subject: [PATCH 89/95] Minor modifications --- Linux Script/configure-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Linux Script/configure-linux.sh b/Linux Script/configure-linux.sh index db1bbac..42ac9dd 100644 --- a/Linux Script/configure-linux.sh +++ b/Linux Script/configure-linux.sh @@ -442,7 +442,7 @@ checkIfConfigurationChanged() done if [ "$ASK_FOR_VERIFICATION" == "true" ]; then - logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file $LOGGLY_RSYSLOG_CONFFILE content is incorrect." + logMsgToConfigSysLog "WARN" "WARN: Loggly rsyslog file /etc/rsyslog.d/22-loggly.conf content has changed." while true; do read -p "Do you wish to override $LOGGLY_RSYSLOG_CONFFILE and re-verify configuration? (yes/no)" yn From 3e0cb26c990e71fd76157281acb8bba59c62dae1 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 30 Sep 2014 23:54:43 +0530 Subject: [PATCH 90/95] Nginx Script: Initial Commit --- Modular Scripts/Nginx/configure-nginx.sh | 342 +++++++++++++++++++++++ 1 file changed, 342 insertions(+) create mode 100644 Modular Scripts/Nginx/configure-nginx.sh diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh new file mode 100644 index 0000000..6f01e54 --- /dev/null +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -0,0 +1,342 @@ +#!/bin/bash + +#downloads configure-linux.sh +echo "INFO: Downloading dependencies - configure-linux.sh" +curl -s -o configure-linux.sh https://www.loggly.com/install/configure-linux.sh +source configure-linux.sh "being-invoked" + +########## Variable Declarations - Start ########## +#name of the current script +SCRIPT_NAME=configure-nginx.sh +#version of the current script +SCRIPT_VERSION=1.0 + +#we have not found the nginx version yet at this point in the script +APP_TAG="\"nginx-version\":\"\"" + +#name of the service, in this case nginx +SERVICE="nginx" +#name of nginx access log file +NGINX_ACCESS_LOG_FILE="access.log" +#name of nginx error log file +NGINX_ERROR_LOG_FILE="error.log" +#name and location of nginx syslog file +NGINX_SYSLOG_CONFFILE=$RSYSLOG_ETCDIR_CONF/21-nginx.conf +#name and location of nginx syslog backup file +NGINX_SYSLOG_CONFFILE_BACKUP=$RSYSLOG_ETCDIR_CONF/21-nginx.conf.loggly.bk + +#this variable will hold the path to the nginx home +LOGGLY_NGINX_HOME= +#this variable will hold the value of the nginx log folder +LOGGLY_NGINX_LOG_HOME= +#this variable will hold the users nginx version +NGINX_VERSION= + +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available at https://www.loggly.com/docs/sending-nginx-logs/" + +#this variable will hold if the check env function for linux is invoked +NGINX_ENV_VALIDATED="false" +########## Variable Declarations - End ########## + +#check if nginx environment is compatible for Loggly +checkNginxLogglyCompatibility() +{ + #check if the linux environment is compatible for Loggly + checkLinuxLogglyCompatibility + + #check if nginx is installed on unix system + checkNginxDetails + + NGINX_ENV_VALIDATED="true" +} + + +# executing the script for loggly to install and configure syslog. +installLogglyConfForNginx() +{ + #log message indicating starting of Loggly configuration + logMsgToConfigSysLog "INFO" "INFO: Initiating Configure Loggly for Nginx." + + #check if nginx environment is compatible with Loggly + if [ "$NGINX_ENV_VALIDATED" = "false" ]; then + checkNginxLogglyCompatibility + fi + + #configure loggly for Linux + installLogglyConf + + #create 21nginx.conf file + write21NginxConfFile + + #check for the nginx log file size + checkLogFileSize $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE + + #verify if the nginx logs made it to loggly + checkIfNginxLogsMadeToLoggly + + #log success message + logMsgToConfigSysLog "SUCCESS" "SUCCESS: Nginx successfully configured to send logs via Loggly." +} + +#executing script to remove loggly configuration for Nginx +removeLogglyConfForNginx() +{ + logMsgToConfigSysLog "INFO" "INFO: Initiating rollback." + + #check if the user has root permission to run this script + checkIfUserHasRootPrivileges + + #check if the OS is supported by the script. If no, then exit + checkIfSupportedOS + + #check if nginx is installed on unix system + checkNginxDetails + + #remove 21nginx.conf file + remove21NginxConfFile + + logMsgToConfigSysLog "INFO" "INFO: Rollback completed." +} + +#identify if nginx is installed on your system and is available as a service +checkNginxDetails() +{ + #verify if nginx is installed as service + if [ ! -f /etc/init.d/$SERVICE ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Nginx is not configured as a service" + exit 1 + fi + + #get the version of nginx installed + getNginxVersion + + #set all the required nginx variables by this script + setNginxVariables +} + + +#sets nginx variables which will be used across various functions +setNginxVariables() +{ + LOGGLY_NGINX_LOG_HOME=/var/log/$SERVICE +} + +#gets the version of nginx installed on the unix box +getNginxVersion() +{ + NGINX_VERSION=$(nginx -v 2>&1) + NGINX_VERSION=${NGINX_VERSION#*/} + APP_TAG="\"nginx-version\":\"$NGINX_VERSION\"" + logMsgToConfigSysLog "INFO" "INFO: nginx version: $NGINX_VERSION" +} + +checkLogFileSize() +{ + accessFileSize=$(wc -c "$1" | cut -f 1 -d ' ') + errorFileSize=$(wc -c "$2" | cut -f 1 -d ' ') + fileSize=$((accessFileSize+errorFileSize)) + if [ $fileSize -ge 102400000 ]; then + logMsgToConfigSysLog "INFO" "INFO: " + while true; do + read -p "WARN: There are currently large log files which may use up your allowed volume. Please rotate your logs before continuing. Would you like to continue now anyway? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Continuing with nginx Loggly configuration."; + break;; + [Nn]* ) + logMsgToConfigSysLog "INFO" "INFO: Current nginx logs size is $fileSize bytes. Discontinuing with nginx Loggly configuration." + exit 1 + break;; + * ) echo "Please answer yes or no.";; + esac + done + elif [ $fileSize -eq 0 ]; then + logMsgToConfigSysLog "WARN" "WARN: There are no recent logs from nginx there so won't be any sent to Loggly. You can generate some logs by visiting a page on your web server." + exit 1 + fi +} + +write21NginxConfFile() +{ + #Create nginx syslog config file if it doesn't exist + echo "INFO: Checking if nginx sysconf file $NGINX_SYSLOG_CONFFILE exist." + if [ -f "$NGINX_SYSLOG_CONFFILE" ]; then + logMsgToConfigSysLog "WARN" "WARN: nginx syslog file $NGINX_SYSLOG_CONFFILE already exist." + while true; do + read -p "Do you wish to override $NGINX_SYSLOG_CONFFILE? (yes/no)" yn + case $yn in + [Yy]* ) + logMsgToConfigSysLog "INFO" "INFO: Going to back up the conf file: $NGINX_SYSLOG_CONFFILE to $NGINX_SYSLOG_CONFFILE_BACKUP"; + sudo mv -f $NGINX_SYSLOG_CONFFILE $NGINX_SYSLOG_CONFFILE_BACKUP; + write21NginxFileContents; + break;; + [Nn]* ) break;; + * ) echo "Please answer yes or no.";; + esac + done + else + write21NginxFileContents + fi +} + +#function to write the contents of nginx syslog config file +write21NginxFileContents() +{ + logMsgToConfigSysLog "INFO" "INFO: Creating file $NGINX_SYSLOG_CONFFILE" + sudo touch $NGINX_SYSLOG_CONFFILE + sudo chmod o+w $NGINX_SYSLOG_CONFFILE + + imfileStr="\$ModLoad imfile + \$InputFilePollInterval 10 + \$WorkDirectory $RSYSLOG_DIR + " + if [[ "$LINUX_DIST" == *"Ubuntu"* ]]; then + imfileStr+="\$PrivDropToGroup adm + " + fi + + imfileStr+=" + # nginx access file: + \$InputFileName $LOGGLY_NGINX_LOG_HOME/$NGINX_ACCESS_LOG_FILE + \$InputFileTag nginx-access: + \$InputFileStateFile stat-nginx-access + \$InputFileSeverity info + \$InputFilePersistStateInterval 20000 + \$InputRunFileMonitor + + #nginx Error file: + \$InputFileName $LOGGLY_NGINX_LOG_HOME/$NGINX_ERROR_LOG_FILE + \$InputFileTag nginx-error: + \$InputFileStateFile stat-nginx-error + \$InputFileSeverity error + \$InputFilePersistStateInterval 20000 + \$InputRunFileMonitor + + #Add a tag for nginx events + \$template LogglyFormatNginx,\"<%pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [$LOGGLY_AUTH_TOKEN@41058 tag=\\\"nginx\\\"] %msg%\n\" + + if \$programname == 'nginx-access' then @@logs-01.loggly.com:514;LogglyFormatNginx + if \$programname == 'nginx-access' then ~ + if \$programname == 'nginx-error' then @@logs-01.loggly.com:514;LogglyFormatNginx + if \$programname == 'nginx-error' then ~ + " + + #change the nginx-21 file to variable from above and also take the directory of the nginx log file. +sudo cat << EOIPFW >> $NGINX_SYSLOG_CONFFILE +$imfileStr +EOIPFW + + restartRsyslog +} + + +#checks if the nginx logs made to loggly +checkIfNginxLogsMadeToLoggly() +{ + counter=1 + maxCounter=10 + + nginxInitialLogCount=0 + nginxLatestLogCount=0 + queryParam="tag%3Anginx&from=-15m&until=now&size=1" + + queryUrl="$LOGGLY_ACCOUNT_URL/apiv2/search?q=$queryParam" + logMsgToConfigSysLog "INFO" "INFO: Search URL: $queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Getting initial nginx log count." + #get the initial count of nginx logs for past 15 minutes + searchAndFetch nginxInitialLogCount "$queryUrl" + + logMsgToConfigSysLog "INFO" "INFO: Verifying if the nginx logs made it to Loggly." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + #get the final count of nginx logs for past 15 minutes + searchAndFetch nginxLatestLogCount "$queryUrl" + let counter=$counter+1 + + while [ "$nginxLatestLogCount" -le "$nginxInitialLogCount" ]; do + echo "INFO: Did not find the test log message in Loggly's search yet. Waiting for 30 secs." + sleep 30 + echo "INFO: Done waiting. Verifying again." + logMsgToConfigSysLog "INFO" "INFO: Verification # $counter of total $maxCounter." + searchAndFetch nginxLatestLogCount "$queryUrl" + let counter=$counter+1 + if [ "$counter" -gt "$maxCounter" ]; then + logMsgToConfigSysLog "ERROR" "ERROR: Nginx logs did not make to Loggly in time. Please check network and firewall settings and retry." + exit 1 + fi + done + + if [ "$nginxLatestLogCount" -gt "$nginxInitialLogCount" ]; then + logMsgToConfigSysLog "SUCCESS" "SUCCESS: nginx logs successfully transferred to Loggly! You are now sending nginx logs to Loggly." + exit 0 + fi +} + +#remove 21nginx.conf file +remove21NginxConfFile() +{ + echo "INFO: Deleting the loggly nginx syslog conf file." + if [ -f "$NGINX_SYSLOG_CONFFILE" ]; then + sudo rm -rf "$NGINX_SYSLOG_CONFFILE" + fi + echo "INFO: Removed all the modified files." + restartRsyslog +} + +#display usage syntax +usage() +{ +cat << EOF +usage: configure-nginx [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] +usage: configure-nginx [-a loggly auth account or subdomain] [-r to rollback] +usage: configure-nginx [-h for help] +EOF +} + +########## Get Inputs from User - Start ########## + +if [ $# -eq 0 ]; then + usage + exit +else +while [ "$1" != "" ]; do + case $1 in + -t | --token ) shift + LOGGLY_AUTH_TOKEN=$1 + echo "AUTH TOKEN $LOGGLY_AUTH_TOKEN" + ;; + -a | --account ) shift + LOGGLY_ACCOUNT=$1 + echo "Loggly account or subdomain: $LOGGLY_ACCOUNT" + ;; + -u | --username ) shift + LOGGLY_USERNAME=$1 + echo "Username is set" + ;; + -p | --password ) shift + LOGGLY_PASSWORD=$1 + ;; + -r | --rollback ) + LOGGLY_ROLLBACK="true" + ;; + -h | --help) + usage + exit + ;; + esac + shift +done +fi + +if [ "$LOGGLY_ACCOUNT" != "" -a "$LOGGLY_USERNAME" != "" ]; then + if [ "$LOGGLY_PASSWORD" = "" ]; then + getPassword + fi + installLogglyConfForNginx +elif [ "$LOGGLY_ROLLBACK" != "" -a "$LOGGLY_ACCOUNT" != "" ]; then + removeLogglyConfForNginx +else + usage +fi + +########## Get Inputs from User - End ########## From 03c5b2d06993c2eaad2fe5c47379ecb2a5acd95a Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 30 Sep 2014 23:57:48 +0530 Subject: [PATCH 91/95] Update README.md --- Modular Scripts/Apache2/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Apache2/README.md b/Modular Scripts/Apache2/README.md index 1b7f393..f24dd8b 100644 --- a/Modular Scripts/Apache2/README.md +++ b/Modular Scripts/Apache2/README.md @@ -3,7 +3,7 @@ Apache Script Configure your Apache server to send logs from access file and error file to Loggly - sudo bash configure-apache -a SUBDOMAIN -u USERNAME + sudo bash configure-apache.sh -a SUBDOMAIN -u USERNAME Stop sending your Apache logs to Loggly From ac442f4f57ac8c2a91ce6ad07e4a759b305c269b Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 30 Sep 2014 23:58:44 +0530 Subject: [PATCH 92/95] Create README.md --- Modular Scripts/Nginx/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Modular Scripts/Nginx/README.md diff --git a/Modular Scripts/Nginx/README.md b/Modular Scripts/Nginx/README.md new file mode 100644 index 0000000..abdbd38 --- /dev/null +++ b/Modular Scripts/Nginx/README.md @@ -0,0 +1,10 @@ +Nginx Script +============= + +Configure your Nginx server to send logs from access file and error file to Loggly + + sudo bash configure-nginx.sh -a SUBDOMAIN -u USERNAME + +Stop sending your Nginx logs to Loggly + + sudo bash configure-nginx.sh -a SUBDOMAIN -r From 9cfdb4cf82a3375e6f2c4a74737493c477209f5c Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Wed, 1 Oct 2014 00:00:57 +0530 Subject: [PATCH 93/95] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 62513af..ae135c5 100644 --- a/README.md +++ b/README.md @@ -14,3 +14,4 @@ You can view the **READMEs** of the various installation scripts at the followin 3. File Monitoring Configuration Script 4. Tomcat Configuration Script 5. S3 Bucket Configuration Script +6. Nginx Script From cf1c2486dbce8bfce04bfb29880a74a2d02fb087 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Fri, 3 Oct 2014 16:01:20 +0530 Subject: [PATCH 94/95] Nginx Script: Updated manual configuration link --- Modular Scripts/Nginx/configure-nginx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modular Scripts/Nginx/configure-nginx.sh b/Modular Scripts/Nginx/configure-nginx.sh index 6f01e54..3df7157 100644 --- a/Modular Scripts/Nginx/configure-nginx.sh +++ b/Modular Scripts/Nginx/configure-nginx.sh @@ -32,7 +32,7 @@ LOGGLY_NGINX_LOG_HOME= #this variable will hold the users nginx version NGINX_VERSION= -MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available at https://www.loggly.com/docs/sending-nginx-logs/" +MANUAL_CONFIG_INSTRUCTION="Manual instructions to configure nginx is available at https://www.loggly.com/docs/nginx-server-logs#manual" #this variable will hold if the check env function for linux is invoked NGINX_ENV_VALIDATED="false" From 476a5effbb8d433e3f261be8cb906bb438903b30 Mon Sep 17 00:00:00 2001 From: psquickitjayant Date: Tue, 7 Oct 2014 23:20:03 +0530 Subject: [PATCH 95/95] File Monitoring: Added tag as parameter --- .../File Monitoring/configure-file-monitoring.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Modular Scripts/File Monitoring/configure-file-monitoring.sh b/Modular Scripts/File Monitoring/configure-file-monitoring.sh index aa0f470..18f3fce 100644 --- a/Modular Scripts/File Monitoring/configure-file-monitoring.sh +++ b/Modular Scripts/File Monitoring/configure-file-monitoring.sh @@ -9,7 +9,7 @@ source configure-linux.sh "being-invoked" #name of the current script SCRIPT_NAME=configure-file-monitoring.sh #version of the current script -SCRIPT_VERSION=1.4 +SCRIPT_VERSION=1.5 #file to monitor (contains complete path and file name) provided by user LOGGLY_FILE_TO_MONITOR= @@ -327,7 +327,7 @@ remove21ConfFile() usage() { cat << EOF -usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-l filealias] +usage: configure-file-monitoring [-a loggly auth account or subdomain] [-t loggly token (optional)] [-u username] [-p password (optional)] [-f filename] [-tag filetag (optional)] [-l filealias] usage: configure-file-monitoring [-a loggly auth account or subdomain] [-r to rollback] [-l filealias] usage: configure-file-monitoring [-h for help] EOF @@ -366,8 +366,13 @@ if [ "$1" != "being-invoked" ]; then ;; -l | --filealias ) shift LOGGLY_FILE_TO_MONITOR_ALIAS=$1 + CONF_FILE_FORMAT_NAME=$CONF_FILE_FORMAT_NAME$1 echo "File alias: $LOGGLY_FILE_TO_MONITOR_ALIAS" ;; + -tag| --filetag ) shift + LOGGLY_FILE_TAG=$1 + echo "File tag: $LOGGLY_FILE_TAG" + ;; -h | --help) usage exit