From 139b0b0bb953b9c4317d3df02944cea610d14688 Mon Sep 17 00:00:00 2001 From: test-account-0 Date: Tue, 9 Feb 2016 10:15:51 +0100 Subject: [PATCH 1/4] pcp2graphite improvement: send all metrics at once in pickled mode Before this change "pickled_input" variable was appended and metrics send over and over again. Time to send metrics to graphite dropped for me from 6 minutes to 2 seconds. --- src/pcp2graphite/pcp2graphite.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pcp2graphite/pcp2graphite.py b/src/pcp2graphite/pcp2graphite.py index fcbffcc6a..71ae72d2e 100644 --- a/src/pcp2graphite/pcp2graphite.py +++ b/src/pcp2graphite/pcp2graphite.py @@ -211,15 +211,15 @@ def send(self, timestamp, miv_tuples): pickled_input = [] for (metric, value) in miv_tuples: pickled_input.append((metric, (timestamp, value))) - # protocol=0 in case carbon is running under an - # older python version than we are - pickled_output = pickle.dumps(pickled_input, protocol=0) - header = struct.pack("!L", len(pickled_output)) - msg = header + pickled_output - if self.context.pmDebug(c_api.PM_DEBUG_APPL0): - print ("Sending %s #tuples %d" % - (time.ctime(timestamp), len(pickled_input))) - self.socket.send(msg) + # protocol=0 in case carbon is running under an + # older python version than we are + pickled_output = pickle.dumps(pickled_input, protocol=0) + header = struct.pack("!L", len(pickled_output)) + msg = header + pickled_output + if self.context.pmDebug(c_api.PM_DEBUG_APPL0): + print ("Sending %s #tuples %d" % + (time.ctime(timestamp), len(pickled_input))) + self.socket.send(msg) else: for (metric, value) in miv_tuples: message = ("%s %s %s\n" % (metric, value, timestamp)) From d4ec9b96c05a6b4857999cc85c56e925087b7e5b Mon Sep 17 00:00:00 2001 From: test-account-0 Date: Tue, 9 Feb 2016 11:22:28 +0100 Subject: [PATCH 2/4] pcp2graphite: cPickle instead of pickle Further performance improvement. --- src/pcp2graphite/pcp2graphite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pcp2graphite/pcp2graphite.py b/src/pcp2graphite/pcp2graphite.py index 71ae72d2e..238c44410 100644 --- a/src/pcp2graphite/pcp2graphite.py +++ b/src/pcp2graphite/pcp2graphite.py @@ -206,14 +206,14 @@ def send(self, timestamp, miv_tuples): self.socket = socket.create_connection((self.graphite_host, self.graphite_port)) if self.pickle: - import pickle + import cPickle import struct pickled_input = [] for (metric, value) in miv_tuples: pickled_input.append((metric, (timestamp, value))) # protocol=0 in case carbon is running under an # older python version than we are - pickled_output = pickle.dumps(pickled_input, protocol=0) + pickled_output = cPickle.dumps(pickled_input, protocol=0) header = struct.pack("!L", len(pickled_output)) msg = header + pickled_output if self.context.pmDebug(c_api.PM_DEBUG_APPL0): From 6731fe909eb67dafd384ead73d435a88c4b28ba6 Mon Sep 17 00:00:00 2001 From: test-account-0 Date: Tue, 9 Feb 2016 13:15:22 +0100 Subject: [PATCH 3/4] pcp2graphite: option to specify pickle protocol Another slight performance improvement with protocol 1 and 2. --- src/pcp2graphite/pcp2graphite.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/pcp2graphite/pcp2graphite.py b/src/pcp2graphite/pcp2graphite.py index 238c44410..66bda8e01 100644 --- a/src/pcp2graphite/pcp2graphite.py +++ b/src/pcp2graphite/pcp2graphite.py @@ -51,7 +51,7 @@ def __init__(self): self.socket = None self.sample_count = 0 self.opts = pmapi.pmOptions() - self.opts.pmSetShortOptions("a:O:s:T:g:p:P:u:m:t:h:t:D:LV?") + self.opts.pmSetShortOptions("a:O:s:T:g:p:P:r:u:m:t:h:t:D:LV?") self.opts.pmSetShortUsage("[options] metricname ...") self.opts.pmSetOptionCallback(self.option) self.opts.pmSetOverrideCallback(self.option_override) @@ -81,10 +81,13 @@ def __init__(self): "(e.g. \"MB\", will omit incompatible units)") self.opts.pmSetLongOption("prefix", 1, 'm', '', "prefix for metric names (default \"pcp.\")") + self.opts.pmSetLongOption("pickle-protocol", 1, 'r', 'PROTOCOL', + "graphite pickle protocol (default 0)") self.opts.pmSetLongOptionHelp() self.graphite_host = "localhost" self.graphite_port = 2004 self.pickle = True + self.pickle_protocol = 0 self.prefix = "pcp." self.unitsstr = None self.units = None # pass verbatim by default @@ -137,6 +140,8 @@ def option(self, opt, optarg, index): elif opt == 'p': self.graphite_port = int(optarg if optarg else "2004") self.pickle = True + elif opt == 'r': + self.pickle_protocol = int(optarg if optarg else "0") elif opt == 'P': self.graphite_port = int(optarg if optarg else "2003") self.pickle = False @@ -211,9 +216,7 @@ def send(self, timestamp, miv_tuples): pickled_input = [] for (metric, value) in miv_tuples: pickled_input.append((metric, (timestamp, value))) - # protocol=0 in case carbon is running under an - # older python version than we are - pickled_output = cPickle.dumps(pickled_input, protocol=0) + pickled_output = cPickle.dumps(pickled_input, protocol=self.pickle_protocol) header = struct.pack("!L", len(pickled_output)) msg = header + pickled_output if self.context.pmDebug(c_api.PM_DEBUG_APPL0): From 668793a2d20d208aa1f4076259269e48c7f97a9e Mon Sep 17 00:00:00 2001 From: test-account-0 Date: Tue, 9 Feb 2016 16:20:59 +0100 Subject: [PATCH 4/4] pcp2graphite: workaround for cPickle and Python 3 --- src/pcp2graphite/pcp2graphite.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pcp2graphite/pcp2graphite.py b/src/pcp2graphite/pcp2graphite.py index 66bda8e01..1bf60d63f 100644 --- a/src/pcp2graphite/pcp2graphite.py +++ b/src/pcp2graphite/pcp2graphite.py @@ -211,12 +211,15 @@ def send(self, timestamp, miv_tuples): self.socket = socket.create_connection((self.graphite_host, self.graphite_port)) if self.pickle: - import cPickle + try: + import cPickle as pickle + except: + import pickle import struct pickled_input = [] for (metric, value) in miv_tuples: pickled_input.append((metric, (timestamp, value))) - pickled_output = cPickle.dumps(pickled_input, protocol=self.pickle_protocol) + pickled_output = pickle.dumps(pickled_input, protocol=self.pickle_protocol) header = struct.pack("!L", len(pickled_output)) msg = header + pickled_output if self.context.pmDebug(c_api.PM_DEBUG_APPL0):