From f1c309005b399cdf6ebb5cfa27756aff8ee84f95 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Mon, 26 Feb 2018 20:26:44 +0100 Subject: [PATCH 1/4] logpipe: add support for drop-unmatched flag This patch adds a new flag to log statements, "drop-unmatched" will cause messages to be dropped along a specific log path, should it not match the filter or be dropped by a parser. Normally in such cases, syslog-ng would continue to process the message along alternative paths. This is basically saying that a filter/parse error is final and is to be used by our if statement. Signed-off-by: Balazs Scheidler --- lib/cfg-tree.c | 5 +++++ lib/cfg-tree.h | 9 +++++---- lib/logpipe.h | 11 +++++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/cfg-tree.c b/lib/cfg-tree.c index 6552fb9543..9031d8d79d 100644 --- a/lib/cfg-tree.c +++ b/lib/cfg-tree.c @@ -371,6 +371,8 @@ log_expr_node_lookup_flag(const gchar *flag) return LC_FINAL; else if (strcmp(flag, "flow_control") == 0 || strcmp(flag, "flow-control") == 0) return LC_FLOW_CONTROL; + else if (strcmp(flag, "drop_unmatched") == 0 || strcmp(flag, "drop-unmatched") == 0) + return LC_DROP_UNMATCHED; msg_error("Unknown log statement flag", evt_tag_str("flag", flag)); return 0; } @@ -617,6 +619,9 @@ cfg_tree_propagate_expr_node_properties_to_pipe(LogExprNode *node, LogPipe *pipe if (node->flags & LC_FLOW_CONTROL) pipe->flags |= PIF_HARD_FLOW_CONTROL; + if (node->flags & LC_DROP_UNMATCHED) + pipe->flags |= PIF_DROP_UNMATCHED; + if (!pipe->expr_node) pipe->expr_node = node; } diff --git a/lib/cfg-tree.h b/lib/cfg-tree.h index 14eee052c4..da257dca90 100644 --- a/lib/cfg-tree.h +++ b/lib/cfg-tree.h @@ -32,10 +32,11 @@ const gchar *log_expr_node_get_content_name(gint content); -#define LC_CATCHALL 1 -#define LC_FALLBACK 2 -#define LC_FINAL 4 -#define LC_FLOW_CONTROL 8 +#define LC_CATCHALL 1 +#define LC_FALLBACK 2 +#define LC_FINAL 4 +#define LC_FLOW_CONTROL 8 +#define LC_DROP_UNMATCHED 16 enum { diff --git a/lib/logpipe.h b/lib/logpipe.h index 87352304ab..c24a47c6ee 100644 --- a/lib/logpipe.h +++ b/lib/logpipe.h @@ -49,8 +49,10 @@ #define PIF_BRANCH_FALLBACK 0x0008 #define PIF_BRANCH_PROPERTIES (PIF_BRANCH_FINAL + PIF_BRANCH_FALLBACK) +#define PIF_DROP_UNMATCHED 0x0010 + /* branch starting with this pipe wants hard flow control */ -#define PIF_HARD_FLOW_CONTROL 0x0010 +#define PIF_HARD_FLOW_CONTROL 0x0020 /* this pipe is a source for messages, it is not meant to be used to * forward messages, syslog-ng will only use these pipes for the @@ -58,7 +60,7 @@ * sending messages to these pipes and these are expected to generate * messages "automatically". */ -#define PIF_SOURCE 0x0020 +#define PIF_SOURCE 0x0040 /* private flags range, to be used by other LogPipe instances for their own purposes */ @@ -343,6 +345,11 @@ log_pipe_queue(LogPipe *s, LogMessage *msg, const LogPathOptions *path_options) { log_pipe_forward_msg(s, msg, path_options); } + + if (path_options->matched && !(*path_options->matched) && (s->flags & PIF_DROP_UNMATCHED)) + { + (*path_options->matched) = TRUE; + } } static inline LogPipe * From 9ebd925d7a252ffc39e936fada9077539edc64b7 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sun, 21 Jan 2018 16:00:11 +0100 Subject: [PATCH 2/4] cfg-grammar: add implementation for if {} elif {} else {} blocks We are already able to do conditional evaluation with a bit arcane syntax: junction { channel { filter { EXPR1 }; ... flags(final); }; channel { filter { EXPR2 }; ... flags(final); }; channel { flags(final); }; }; However, this is not very intuitive. This patch adds syntactic sugar using the if / else keywords to achieve the same, such as: if (message('foo')) { ... } elif (message('bar')) { ... } else { ... } We actually have two formats of the conditional expressions: 1) the one with an explicit filter expression if (message('foo')) { parser { date-parser(); }; } else { ... }; 2) the one where the condition is embedded in the log path if { filter { message('foo')); }; parser { date-parser(); }; } else { ... }; The difference between the two is that the second format would consider all filters and all parsers as the condition, combined. The first one only uses the filter expression in if(). The difference in evaluation of the two formats are: 1) if the message contains 'foo' and the date-parser() fails, the message is dropped and execution does not continue along the else path. If if does not contain 'foo' the else is obviously taken. 2) if the message contains 'foo' and the date-parser() fails the else branch is taken, just as if the message does not contain 'foo'. This change should make the configuration for complex of stuff a lot simpler. Signed-off-by: Balazs Scheidler --- lib/cfg-grammar.y | 46 +++++++++- lib/cfg-parser.c | 3 + lib/cfg-tree.c | 211 +++++++++++++++++++++++++++++++++++++++++++++- lib/cfg-tree.h | 4 + 4 files changed, 261 insertions(+), 3 deletions(-) diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index bb02efb9d8..d0115af70d 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -172,10 +172,13 @@ extern struct _StatsOptions *last_stats_options; %token KW_BLOCK 10007 %token KW_JUNCTION 10008 %token KW_CHANNEL 10009 +%token KW_IF 10010 +%token KW_ELSE 10011 +%token KW_ELIF 10012 /* source & destination items */ -%token KW_INTERNAL 10010 -%token KW_FILE 10011 +%token KW_INTERNAL 10020 +%token KW_FILE 10021 %token KW_SYSLOG 10060 /* option items */ @@ -422,6 +425,8 @@ DNSCacheOptions *last_dns_cache_options; %type log_last_junction %type log_junction %type log_content +%type log_conditional +%type log_if %type log_forks %type log_fork @@ -712,6 +717,7 @@ log_item | KW_REWRITE '{' rewrite_content '}' { $$ = log_expr_node_new_rewrite(NULL, $3, &@$); } | KW_DESTINATION '(' string ')' { $$ = log_expr_node_new_destination_reference($3, &@$); free($3); } | KW_DESTINATION '{' dest_content '}' { $$ = log_expr_node_new_destination(NULL, $3, &@$); } + | log_conditional { $$ = $1; } | log_junction { $$ = $1; } ; @@ -742,6 +748,42 @@ log_fork | KW_CHANNEL '{' log_content '}' { $$ = $3; } ; +log_conditional + : log_if { $$ = $1; } + | log_if KW_ELSE '{' log_content '}' + { + log_expr_node_conditional_set_false_branch_of_the_last_if($1, $4); + $$ = $1; + } + ; + +log_if + : KW_IF '(' filter_content ')' '{' log_content '}' + { + $$ = log_expr_node_new_conditional_with_filter($3, $6, &@$); + } + | KW_IF '{' log_content '}' + { + $$ = log_expr_node_new_conditional_with_block($3, &@$); + } + | log_if KW_ELIF '(' filter_content ')' '{' log_content '}' + { + LogExprNode *false_branch; + + false_branch = log_expr_node_new_conditional_with_filter($4, $7, &@$); + log_expr_node_conditional_set_false_branch_of_the_last_if($1, false_branch); + $$ = $1; + } + | log_if KW_ELIF '{' log_content '}' + { + LogExprNode *false_branch; + + false_branch = log_expr_node_new_conditional_with_block($4, &@$); + log_expr_node_conditional_set_false_branch_of_the_last_if($1, false_branch); + $$ = $1; + } + ; + log_content : log_items log_last_junction log_flags { $$ = log_expr_node_new_log(log_expr_node_append_tail($1, $2), $3, &@$); } ; diff --git a/lib/cfg-parser.c b/lib/cfg-parser.c index f077dafc7c..7b41efcadf 100644 --- a/lib/cfg-parser.c +++ b/lib/cfg-parser.c @@ -60,6 +60,9 @@ static CfgLexerKeyword main_keywords[] = { "options", KW_OPTIONS }, { "include", KW_INCLUDE, }, { "block", KW_BLOCK }, + { "if", KW_IF }, + { "else", KW_ELSE }, + { "elif", KW_ELIF }, /* source or destination items */ { "internal", KW_INTERNAL }, diff --git a/lib/cfg-tree.c b/lib/cfg-tree.c index 9031d8d79d..b370d25b23 100644 --- a/lib/cfg-tree.c +++ b/lib/cfg-tree.c @@ -281,7 +281,6 @@ log_expr_node_new_pipe(LogPipe *pipe, YYLTYPE *yylloc) return node; } - LogExprNode * log_expr_node_new_source(const gchar *name, LogExprNode *children, YYLTYPE *yylloc) { @@ -360,6 +359,216 @@ log_expr_node_new_junction(LogExprNode *children, YYLTYPE *yylloc) return log_expr_node_new(ENL_JUNCTION, ENC_PIPE, NULL, children, 0, yylloc); } +/**************************************************************************** + * Functions related to conditional nodes + * + * These are higher-level functions that map if-elif-else structure to + * LogExprNode instances. Rather than using a higher level data struct + * which then generates LogExprNode instances, we represent/manipulate the + * if-elif-else structure right within LogExprNode. + * + * A conditional node is simply a junction with two children: + * + * 1) the first child is the "TRUE" branch, with the filter expression + * attached and the final flag + * + * 2) the second child is the "FALSE" branch, possibly empty, but also + * the final flag. + * + * Basically this is the equivalent to: + * + * junction { + * channel { + * filter { EXPRESSION; }; + * flags(final); + * }; + * channel { + * flags(final); + * }; + * }; + * + * When parsing an if block, we generate both children immediately, with the + * empty 2nd channel, and then if an elif or else comes, the FALSE branch + * gets replaced. + * + * The series of if-elif-else sequences is represented by its first + * LogExprNode (e.g. the first if). When we need to add an else or elif, + * we would have to locate the last dangling if statement based on this + * first LogExprNode. The alternative would have been to store the last if + * statement in a variable, however that becomes pretty complicated if we + * need to handle nesting. + * + ****************************************************************************/ + +static LogExprNode * +log_expr_node_conditional_get_true_branch(LogExprNode *node) +{ + g_assert(node->layout == ENL_JUNCTION); + + LogExprNode *branches = node->children; + + g_assert(branches != NULL); + g_assert(branches->next != NULL); + g_assert(branches->next->next == NULL); + + /* first child */ + return branches; +} + +static LogExprNode * +log_expr_node_conditional_get_false_branch(LogExprNode *node) +{ + g_assert(node->layout == ENL_JUNCTION); + + LogExprNode *branches = node->children; + g_assert(branches != NULL); + g_assert(branches->next != NULL); + g_assert(branches->next->next == NULL); + + /* second child */ + return branches->next; +} + +static gboolean +log_expr_node_conditional_is_branch_empty(LogExprNode *node) +{ + return node->children == NULL; +} + +/* this function locates the last dangling if, based on the very first if + * statement in a series of ifs at the same level */ +static LogExprNode * +_locate_last_conditional_along_nested_else_blocks(LogExprNode *head) +{ + while (1) + { + g_assert(log_expr_node_conditional_get_true_branch(head) != NULL); + g_assert(log_expr_node_conditional_get_false_branch(head) != NULL); + + LogExprNode *false_branch = log_expr_node_conditional_get_false_branch(head); + + /* check if this is the last else */ + if (log_expr_node_conditional_is_branch_empty(false_branch)) + return head; + head = false_branch->children; + } + g_assert_not_reached(); +} + +/* change the FALSE branch (e.g. the else case) of the last dangling if, specified by the head element */ +void +log_expr_node_conditional_set_false_branch_of_the_last_if(LogExprNode *conditional_head_node, LogExprNode *false_expr) +{ + LogExprNode *conditional_node = _locate_last_conditional_along_nested_else_blocks(conditional_head_node); + LogExprNode *branches = conditional_node->children; + + /* a conditional branch always have two children (see the constructor + * below), the first one is the "true" branch and the second one is the + * "false" branch, as they are constructed as final log channels with + * filter statement in the first one as the "if" expression. */ + + /* assert that we only have two children */ + g_assert(branches != NULL); + g_assert(branches->next != NULL); + g_assert(branches->next->next == NULL); + + + /* construct the new false branch */ + LogExprNode *false_branch = log_expr_node_new_log( + false_expr, + log_expr_node_lookup_flag("final"), + NULL + ); + + /* unlink and free the old one */ + LogExprNode *old_false_branch = branches->next; + branches->next = false_branch; + false_branch->parent = conditional_node; + log_expr_node_free(old_false_branch); +} + +/* + */ +LogExprNode * +log_expr_node_new_conditional_with_filter(LogExprNode *filter_pipe, LogExprNode *true_expr, YYLTYPE *yylloc) +{ + LogExprNode *filter_node = log_expr_node_new_filter(NULL, filter_pipe, NULL); + + /* + * channel { + * filter { EXPRESSION }; + * true_expr; + * flags(final); + * }; + */ + LogExprNode *true_branch = log_expr_node_new_log( + log_expr_node_append_tail( + filter_node, + log_expr_node_new_log(true_expr, LC_DROP_UNMATCHED, NULL) + ), + LC_FINAL, + NULL + ); + + /* + * channel { + * flags(final); + * }; + * + * NOTE: the false branch may be modified later, once an else or elif is + * encountered in the configuration, see + * log_expr_node_conditional_set_false_branch_of_the_last_if() function + * above. + */ + LogExprNode *false_branch = log_expr_node_new_log( + NULL, + LC_FINAL, + NULL + ); + return log_expr_node_new_junction( + log_expr_node_append_tail(true_branch, false_branch), + yylloc + ); +} + +LogExprNode * +log_expr_node_new_conditional_with_block(LogExprNode *block, YYLTYPE *yylloc) +{ + /* + * channel { + * filtering_and_parsing_expr; + * flags(final); + * }; + */ + LogExprNode *true_branch = log_expr_node_new_log( + block, + LC_FINAL, + NULL + ); + + /* + * channel { + * flags(final); + * }; + * + * NOTE: the false branch may be modified later, once an else or elif is + * encountered in the configuration, see + * log_expr_node_conditional_set_false_branch_of_the_last_if() function + * above. + */ + LogExprNode *false_branch = log_expr_node_new_log( + NULL, + LC_FINAL, + NULL + ); + return log_expr_node_new_junction( + log_expr_node_append_tail(true_branch, false_branch), + yylloc + ); +} + +/****************************************************************************/ + gint log_expr_node_lookup_flag(const gchar *flag) { diff --git a/lib/cfg-tree.h b/lib/cfg-tree.h index da257dca90..39f8170fef 100644 --- a/lib/cfg-tree.h +++ b/lib/cfg-tree.h @@ -148,6 +148,10 @@ LogExprNode *log_expr_node_new_rewrite_reference(const gchar *name, YYLTYPE *yyl LogExprNode *log_expr_node_new_log(LogExprNode *children, guint32 flags, YYLTYPE *yylloc); LogExprNode *log_expr_node_new_sequence(LogExprNode *children, YYLTYPE *yylloc); LogExprNode *log_expr_node_new_junction(LogExprNode *children, YYLTYPE *yylloc); +void log_expr_node_conditional_set_false_branch_of_the_last_if(LogExprNode *conditional_node, LogExprNode *false_expr); +LogExprNode *log_expr_node_new_conditional_with_filter(LogExprNode *filter_pipe, LogExprNode *true_expr, + YYLTYPE *yylloc); +LogExprNode *log_expr_node_new_conditional_with_block(LogExprNode *block, YYLTYPE *yylloc); typedef struct _CfgTree { From f613abc72e1d88b64d67ad07903fc35d022c9613 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Sat, 27 Jan 2018 18:54:12 +0100 Subject: [PATCH 3/4] scl/cisco: use if/elif in cisco-timestamp-parser() Instead of using a complicated junction/channel construct, use the more descriptive if, making the parser a lot easer to read. Signed-off-by: Balazs Scheidler --- scl/cisco/plugin.conf | 67 +++++++++++-------------------------------- 1 file changed, 17 insertions(+), 50 deletions(-) diff --git a/scl/cisco/plugin.conf b/scl/cisco/plugin.conf index c6f524a7b4..427fd91192 100644 --- a/scl/cisco/plugin.conf +++ b/scl/cisco/plugin.conf @@ -42,29 +42,16 @@ block parser cisco-timestamp-parser(template()) { channel { rewrite { set("`template`" value('1')); - }; - junction { - # alternative #1, no year - channel { - filter { - match('^[.*]?([A-Za-z]{3} [0-9 ]\d \d{2}:\d{2}:\d{2})' value('1') flags(store-matches)); - }; - parser { - date-parser(format('%b %d %H:%M:%S') template("$1")); - }; - flags(final); - }; + }; - channel { - filter { - match('^[.*]?([A-Za-z]{3} [0-9 ]\d \d{4} \d{2}:\d{2}:\d{2})' value('1') flags(store-matches)); - }; - parser { - date-parser(format('%b %d %Y %H:%M:%S') template("$1")); - }; - flags(final); - }; + if { + filter { match('^[.*]?([A-Za-z]{3} [0-9 ]\d \d{2}:\d{2}:\d{2})' value('1') flags(store-matches)); }; + parser { date-parser(format('%b %d %H:%M:%S') template("$1")); }; + } else { + filter { match('^[.*]?([A-Za-z]{3} [0-9 ]\d \d{4} \d{2}:\d{2}:\d{2})' value('1') flags(store-matches)); }; + parser { date-parser(format('%b %d %Y %H:%M:%S') template("$1")); }; }; + rewrite { unset(value('1')); }; @@ -91,35 +78,15 @@ block parser cisco-parser(prefix(".cisco.")) { }; - junction { - channel { - parser { - cisco-timestamp-parser(template("$1")); - }; - flags(final); - }; - channel { - filter { - match("^(?'HOST'[^:]+): (.*)" value('1') flags(store-matches) type(pcre)); - }; - parser { - cisco-timestamp-parser(template("$2")); - }; - flags(final); - }; - - channel { - filter { - match("^(?'HOST'[^:]+)$" value('1') flags(store-matches) type(pcre)); - }; - flags(final); - }; - channel { - filter { - match("^$" value('1') flags(store-matches) type(pcre)); - }; - flags(final); - }; + if { + parser { cisco-timestamp-parser(template("$1")); }; + } elif { + filter { match("^(?'HOST'[^:]+): (.*)" value('1') flags(store-matches) type(pcre)); }; + parser { cisco-timestamp-parser(template("$2")); }; + } elif { + filter { match("^(?'HOST'[^:]+)$" value('1') flags(store-matches) type(pcre)); }; + } else { + filter { match("^$" value('1') flags(store-matches) type(pcre)); }; }; rewrite { unset(value("1")); From 9339ecb91b001528c47ff055f021d450bba30353 Mon Sep 17 00:00:00 2001 From: Balazs Scheidler Date: Mon, 26 Feb 2018 21:36:25 +0100 Subject: [PATCH 4/4] scl/solaris: simplify configuration using if expressions Signed-off-by: Balazs Scheidler --- scl/solaris/plugin.conf | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/scl/solaris/plugin.conf b/scl/solaris/plugin.conf index 0aaafbc215..a1619172b7 100644 --- a/scl/solaris/plugin.conf +++ b/scl/solaris/plugin.conf @@ -21,32 +21,29 @@ # ############################################################################# +# Sample message: +# <55>2018-02-26T10:00:00 host sshd: [ID 800047 auth.info] Accepted publickey for acsss from 192.168.1.99 port 40386 ssh2 + block parser extract-solaris-msgid() { - channel { - junction { - channel { - filter { message("[ID" type(string) flags(prefix)); }; - parser { - csv-parser( - columns("0", "MESSAGE") - delimiters(" ") - flags(greedy) - quote-pairs('[]') - ); + channel { + if { + filter { message("[ID" type(string) flags(prefix)); }; + parser { + csv-parser( + columns("0", "MESSAGE") + delimiters(" ") + flags(greedy) + quote-pairs('[]') + ); - csv-parser( - columns("", ".solaris.msgid", "") - template("$0") - delimiters(" ") - ); + csv-parser( + columns("", ".solaris.msgid", "") + template("$0") + delimiters(" ") + ); + }; }; - flags(final); - }; - channel { - flags(final); - }; }; - }; };