From ebca67d174f15bf096a5c7eaa515d14a840bc44a Mon Sep 17 00:00:00 2001 From: Don Johnson Date: Sat, 9 Jul 2011 16:46:10 -0700 Subject: [PATCH 1/3] adding types.db list from collectd v5.0.0 --- types.db | 187 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 types.db diff --git a/types.db b/types.db new file mode 100644 index 0000000..e6345ab --- /dev/null +++ b/types.db @@ -0,0 +1,187 @@ +absolute value:ABSOLUTE:0:U +apache_bytes value:DERIVE:0:U +apache_connections value:GAUGE:0:65535 +apache_idle_workers value:GAUGE:0:65535 +apache_requests value:DERIVE:0:U +apache_scoreboard value:GAUGE:0:65535 +ath_nodes value:GAUGE:0:65535 +ath_stat value:DERIVE:0:U +bitrate value:GAUGE:0:4294967295 +bytes value:GAUGE:0:U +cache_operation value:DERIVE:0:U +cache_ratio value:GAUGE:0:100 +cache_result value:DERIVE:0:U +cache_size value:GAUGE:0:4294967295 +charge value:GAUGE:0:U +compression_ratio value:GAUGE:0:2 +compression uncompressed:DERIVE:0:U, compressed:DERIVE:0:U +connections value:DERIVE:0:U +conntrack value:GAUGE:0:4294967295 +contextswitch value:DERIVE:0:U +counter value:COUNTER:U:U +cpufreq value:GAUGE:0:U +cpu value:DERIVE:0:U +current_connections value:GAUGE:0:U +current_sessions value:GAUGE:0:U +current value:GAUGE:U:U +delay value:GAUGE:-1000000:1000000 +derive value:DERIVE:0:U +df_complex value:GAUGE:0:U +df_inodes value:GAUGE:0:U +df used:GAUGE:0:1125899906842623, free:GAUGE:0:1125899906842623 +disk_latency read:GAUGE:0:U, write:GAUGE:0:U +disk_merged read:DERIVE:0:U, write:DERIVE:0:U +disk_octets read:DERIVE:0:U, write:DERIVE:0:U +disk_ops_complex value:DERIVE:0:U +disk_ops read:DERIVE:0:U, write:DERIVE:0:U +disk_time read:DERIVE:0:U, write:DERIVE:0:U +dns_answer value:DERIVE:0:U +dns_notify value:DERIVE:0:U +dns_octets queries:DERIVE:0:U, responses:DERIVE:0:U +dns_opcode value:DERIVE:0:U +dns_qtype_cached value:GAUGE:0:4294967295 +dns_qtype value:DERIVE:0:U +dns_query value:DERIVE:0:U +dns_question value:DERIVE:0:U +dns_rcode value:DERIVE:0:U +dns_reject value:DERIVE:0:U +dns_request value:DERIVE:0:U +dns_resolver value:DERIVE:0:U +dns_response value:DERIVE:0:U +dns_transfer value:DERIVE:0:U +dns_update value:DERIVE:0:U +dns_zops value:DERIVE:0:U +email_check value:GAUGE:0:U +email_count value:GAUGE:0:U +email_size value:GAUGE:0:U +entropy value:GAUGE:0:4294967295 +fanspeed value:GAUGE:0:U +file_size value:GAUGE:0:U +files value:GAUGE:0:U +fork_rate value:DERIVE:0:U +frequency value:GAUGE:0:U +frequency_offset value:GAUGE:-1000000:1000000 +fscache_stat value:DERIVE:0:U +gauge value:GAUGE:U:U +http_request_methods value:DERIVE:0:U +http_requests value:DERIVE:0:U +http_response_codes value:DERIVE:0:U +humidity value:GAUGE:0:100 +if_collisions value:DERIVE:0:U +if_dropped rx:DERIVE:0:U, tx:DERIVE:0:U +if_errors rx:DERIVE:0:U, tx:DERIVE:0:U +if_multicast value:DERIVE:0:U +if_octets rx:DERIVE:0:U, tx:DERIVE:0:U +if_packets rx:DERIVE:0:U, tx:DERIVE:0:U +if_rx_errors value:DERIVE:0:U +if_tx_errors value:DERIVE:0:U +invocations value:DERIVE:0:U +io_octets rx:DERIVE:0:U, tx:DERIVE:0:U +io_packets rx:DERIVE:0:U, tx:DERIVE:0:U +ipt_bytes value:DERIVE:0:U +ipt_packets value:DERIVE:0:U +irq value:DERIVE:0:U +latency value:GAUGE:0:65535 +links value:GAUGE:0:U +load shortterm:GAUGE:0:100, midterm:GAUGE:0:100, longterm:GAUGE:0:100 +memcached_command value:DERIVE:0:U +memcached_connections value:GAUGE:0:U +memcached_items value:GAUGE:0:U +memcached_octets rx:DERIVE:0:U, tx:DERIVE:0:U +memcached_ops value:DERIVE:0:U +memory value:GAUGE:0:281474976710656 +multimeter value:GAUGE:U:U +mysql_commands value:DERIVE:0:U +mysql_handler value:DERIVE:0:U +mysql_locks value:DERIVE:0:U +mysql_log_position value:DERIVE:0:U +mysql_octets rx:DERIVE:0:U, tx:DERIVE:0:U +nfs_procedure value:DERIVE:0:U +nginx_connections value:GAUGE:0:U +nginx_requests value:DERIVE:0:U +node_octets rx:DERIVE:0:U, tx:DERIVE:0:U +node_rssi value:GAUGE:0:255 +node_stat value:DERIVE:0:U +node_tx_rate value:GAUGE:0:127 +operations value:DERIVE:0:U +percent value:GAUGE:0:100.1 +pg_blks value:DERIVE:0:U +pg_db_size value:GAUGE:0:U +pg_n_tup_c value:DERIVE:0:U +pg_n_tup_g value:GAUGE:0:U +pg_numbackends value:GAUGE:0:U +pg_scan value:DERIVE:0:U +pg_xact value:DERIVE:0:U +ping_droprate value:GAUGE:0:100 +ping value:GAUGE:0:65535 +ping_stddev value:GAUGE:0:65535 +players value:GAUGE:0:1000000 +power value:GAUGE:0:U +protocol_counter value:DERIVE:0:U +ps_code value:GAUGE:0:9223372036854775807 +ps_count processes:GAUGE:0:1000000, threads:GAUGE:0:1000000 +ps_cputime user:DERIVE:0:U, syst:DERIVE:0:U +ps_data value:GAUGE:0:9223372036854775807 +ps_disk_octets read:DERIVE:0:U, write:DERIVE:0:U +ps_disk_ops read:DERIVE:0:U, write:DERIVE:0:U +ps_pagefaults minflt:DERIVE:0:U, majflt:DERIVE:0:U +ps_rss value:GAUGE:0:9223372036854775807 +ps_stacksize value:GAUGE:0:9223372036854775807 +ps_state value:GAUGE:0:65535 +ps_vm value:GAUGE:0:9223372036854775807 +queue_length value:GAUGE:0:U +records value:GAUGE:0:U +requests value:GAUGE:0:U +response_time value:GAUGE:0:U +route_etx value:GAUGE:0:U +route_metric value:GAUGE:0:U +routes value:GAUGE:0:U +serial_octets rx:DERIVE:0:U, tx:DERIVE:0:U +signal_noise value:GAUGE:U:0 +signal_power value:GAUGE:U:0 +signal_quality value:GAUGE:0:U +snr value:GAUGE:0:U +spam_check value:GAUGE:0:U +spam_score value:GAUGE:U:U +swap_io value:DERIVE:0:U +swap value:GAUGE:0:1099511627776 +tcp_connections value:GAUGE:0:4294967295 +temperature value:GAUGE:-273.15:U +threads value:GAUGE:0:U +time_dispersion value:GAUGE:-1000000:1000000 +timeleft value:GAUGE:0:3600 +time_offset value:GAUGE:-1000000:1000000 +total_bytes value:DERIVE:0:U +total_connections value:DERIVE:0:U +total_operations value:DERIVE:0:U +total_requests value:DERIVE:0:U +total_sessions value:DERIVE:0:U +total_threads value:DERIVE:0:U +total_time_in_ms value:DERIVE:0:U +total_values value:DERIVE:0:U +uptime value:GAUGE:0:4294967295 +users value:GAUGE:0:65535 +vcpu value:GAUGE:0:U +virt_cpu_total value:DERIVE:0:U +virt_vcpu value:DERIVE:0:U +vmpage_action value:DERIVE:0:U +vmpage_faults minflt:DERIVE:0:U, majflt:DERIVE:0:U +vmpage_io in:DERIVE:0:U, out:DERIVE:0:U +vmpage_number value:GAUGE:0:4294967295 +volatile_changes value:GAUGE:0:U +voltage_threshold value:GAUGE:U:U, threshold:GAUGE:U:U +voltage value:GAUGE:U:U +vs_memory value:GAUGE:0:9223372036854775807 +vs_processes value:GAUGE:0:65535 +vs_threads value:GAUGE:0:65535 +# +# Legacy types +# (required for the v5 upgrade target) +# +arc_counts demand_data:COUNTER:0:U, demand_metadata:COUNTER:0:U, prefetch_data:COUNTER:0:U, prefetch_metadata:COUNTER:0:U +arc_l2_bytes read:COUNTER:0:U, write:COUNTER:0:U +arc_l2_size value:GAUGE:0:U +arc_ratio value:GAUGE:0:U +arc_size current:GAUGE:0:U, target:GAUGE:0:U, minlimit:GAUGE:0:U, maxlimit:GAUGE:0:U +mysql_qcache hits:COUNTER:0:U, inserts:COUNTER:0:U, not_cached:COUNTER:0:U, lowmem_prunes:COUNTER:0:U, queries_in_cache:GAUGE:0:U +mysql_threads running:GAUGE:0:U, connected:GAUGE:0:U, cached:GAUGE:0:U, created:COUNTER:0:U From 6ad01f67c77681c611a0da3787b615cfef8d0243 Mon Sep 17 00:00:00 2001 From: Don Johnson Date: Sat, 9 Jul 2011 16:47:55 -0700 Subject: [PATCH 2/3] adding support for high-precision timestamps, multiple value types in each metric, and types.db metric name lookups --- collectd-graphite-proxy.js | 60 +++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/collectd-graphite-proxy.js b/collectd-graphite-proxy.js index 6b8d018..9430693 100644 --- a/collectd-graphite-proxy.js +++ b/collectd-graphite-proxy.js @@ -15,10 +15,35 @@ * This code below will then convert the PUTVAL statements to graphite metrics * and ship them to 'monitor:2003' */ - var http = require("http"); var net = require("net"); var assert = require("assert"); +var fs = require('fs'); + +var types = fs.readFileSync('./types.db', encoding='utf8').split("\n"); + +var typesObj = new Object; + +var type_comments_re = /^#/; +var type_cut_re = /^([^\s]+)\s+(.*)/; + +for (var i in types) { + if (!type_comments_re.exec(types[i])) { + typeSet = type_cut_re.exec(types[i]) + if (!typeSet) { continue; } + for (var t=0;t < typeSet.length;t++) { + var name = typeSet[1]; + typesObj[name] = new Array(); + var eachType = typeSet[2].split(", ") + for (var u=0; u < eachType.length; u++){ + var theName = eachType[u].split(":")[0]; + typesObj[name].push(theName); + } + } + } +} + + try { var graphite_connection = net.createConnection(2003, host=process.argv[2]); @@ -33,23 +58,36 @@ graphite_connection.on("error", function() { }); var request_handler = function(request, response) { - var putval_re = /^PUTVAL ([^ ]+)(?: ([^ ]+=[^ ]+)?) ([0-9]+)(:(?:-?[0-9.]+)+)/; + var putval_re = /^PUTVAL ([^ ]+)(?: ([^ ]+=[^ ]+)?) ([0-9.]+)(:.*)/; request.addListener("data", function(chunk) { - metrics = chunk.toString().split("\n") + metrics = chunk.toString().split("\n"); for (var i in metrics) { var m = putval_re.exec(metrics[i]); if (!m) { continue; } - var name = m[1]; - var options = m[2]; - var time = m[3]; - var value = m[4].replace(/:/, ""); + var values = m[4].split(":"); + + for (var v in values) { + + var name = m[1]; + var options = m[2]; + var time = m[3]; + + if ( v == 0 ) { + continue; + } + + name = "agents." + name.replace(/\./g, "_").replace(/\//g, "."); + if ( values.length > 2 ) { + var metric = name.split(".")[3]; + name = name + "_" + typesObj[metric][v - 1]; + } + message = [name, values[v], time].join(" "); + graphite_connection.write(message + "\n"); + + } - name = "collectd." + name.replace(/\./g, "_").replace(/\//g, "."); - message = [name, value, time].join(" ") - console.log(message) - graphite_connection.write(message + "\n"); } }); From e64ccd18381de6e822a9225928cdbd3cc9a94d70 Mon Sep 17 00:00:00 2001 From: Corry Haines Date: Fri, 26 Aug 2011 10:19:50 -0700 Subject: [PATCH 3/3] Improve on @donjohnson's ideas First, we look at the system installed types.db (works on centos package, should probably be generalized). Later, we split up the metric name into is parts according to the collectd data spec. The instance are split out and handled somewhat better. This fixes an issue where type instances will break the lookup in the types.db file. This also fixes an issue where metrics will be poorly named when they appear in graphite. CAVEAT: This makes changes that are not compatible with existing proxy users. The graphs will be given different names. tl:dr This breaks backwards compatability --- collectd-graphite-proxy.js | 51 ++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/collectd-graphite-proxy.js b/collectd-graphite-proxy.js index 9430693..72ac110 100644 --- a/collectd-graphite-proxy.js +++ b/collectd-graphite-proxy.js @@ -20,7 +20,7 @@ var net = require("net"); var assert = require("assert"); var fs = require('fs'); -var types = fs.readFileSync('./types.db', encoding='utf8').split("\n"); +var types = fs.readFileSync('/usr/share/collectd/types.db', encoding='utf8').split("\n"); var typesObj = new Object; @@ -44,7 +44,6 @@ for (var i in types) { } - try { var graphite_connection = net.createConnection(2003, host=process.argv[2]); } catch (error) { @@ -78,10 +77,52 @@ var request_handler = function(request, response) { continue; } - name = "agents." + name.replace(/\./g, "_").replace(/\//g, "."); + // Replace some chars for graphite, split into parts + var name_parts = name.replace(/\./g, "_").replace(/\//g, ".").split("."); + + // Start to construct the new name + var rebuild = ["agents"] + + var host = name_parts[0].split(/_/)[0] + rebuild = rebuild.concat(host) + + // Pluigin names can contain an "instance" which is set apart by a dash + var plugin = name_parts[1].split("-") + rebuild = rebuild.concat(plugin[0]) + if (plugin.length > 1) { + var plugin_instance = plugin.slice(1).join("-") + rebuild = rebuild.concat(plugin_instance) + } + plugin = plugin[0] + + // Type names can also contain an "instance" + var type = name_parts[2].split("-") + if (type[0] != plugin) { + // If type and plugin are equal, delete one to clean up a bit + rebuild = rebuild.concat(type[0]) + } + if (type.length > 1) { + var type_instance = type.slice(1).join("-") + rebuild = rebuild.concat(type_instance) + } + type = type[0] + + // Put the name back together + name = rebuild.join(".") + if ( values.length > 2 ) { - var metric = name.split(".")[3]; - name = name + "_" + typesObj[metric][v - 1]; + var metric = name_parts[2]; + + // If the metric contains a '-' (after removing the instance name) + // then we want to remove it before looking up in the types.db + index = metric.search(/-/) + if (index > -1) { + metric = /^([\w]+)-(.*)$/.exec(metric); + } else { + // Kinda a hack + metric = [ "", metric] + } + name = name + "." + typesObj[metric[1]][v - 1]; } message = [name, values[v], time].join(" "); graphite_connection.write(message + "\n");