diff --git a/client.go b/client.go index 71f3807..81d0f04 100644 --- a/client.go +++ b/client.go @@ -495,6 +495,8 @@ func (c *PCPClient) writeMetrics() { launchInstanceMetric(metric.pcpInstanceMetric) case *PCPGaugeVector: launchInstanceMetric(metric.pcpInstanceMetric) + case *PCPHistogram: + launchInstanceMetric(metric.pcpInstanceMetric) } } diff --git a/client_test.go b/client_test.go index c4b5869..95a244c 100644 --- a/client_test.go +++ b/client_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/codahale/hdrhistogram" "github.com/performancecopilot/speed/mmvdump" ) @@ -1122,3 +1123,74 @@ func TestGaugeVector(t *testing.T) { t.Errorf("cannot retrieve m.1[m2] value, error: %v", err) } } + +func TestHistogram(t *testing.T) { + hist := hdrhistogram.New(0, 100, 5) + + h, err := NewPCPHistogram("test.hist", 0, 100, 5) + if err != nil { + t.Fatalf("cannot create metric, error: %v", err) + } + + c, err := NewPCPClient("test") + if err != nil { + t.Fatalf("cannot create client, error: %v", err) + } + + c.MustRegister(h) + + c.MustStart() + defer c.MustStop() + + _, _, m, v, i, id, s, err := mmvdump.Dump(c.writer.Bytes()) + if err != nil { + t.Fatalf("cannot create dump, error: %v", err) + } + + matchMetricsAndValues(m, v, i, s, c, t) + matchInstancesAndInstanceDomains(i, id, s, c, t) + + for i := int64(1); i <= 100; i++ { + err = hist.RecordValues(i, i) + if err != nil { + t.Errorf("hdrhistogram couldn't record, error: %v", err) + } + + h.MustRecordN(i, i) + } + + _, _, m, v, _, _, _, err = mmvdump.Dump(c.writer.Bytes()) + if err != nil { + t.Fatalf("cannot create dump, error: %v", err) + } + + cases := [...]struct { + ins string + val float64 + }{ + {"mean", hist.Mean()}, + {"mean", h.Mean()}, + + {"variance", math.Pow(hist.StdDev(), 2)}, + {"variance", h.Variance()}, + + {"standard_deviation", hist.StdDev()}, + {"standard_deviation", h.StandardDeviation()}, + + {"max", float64(hist.Max())}, + {"max", float64(h.Max())}, + + {"min", float64(hist.Min())}, + {"min", float64(h.Min())}, + } + + for _, c := range cases { + off := h.indom.instances[c.ins].offset + moff, _ := findMetric(h, m) + _, dv := findInstanceValue(moff, uint64(off), v) + val, _ := mmvdump.FixedVal(uint64(dv.Val), mmvdump.DoubleType) + if c.val != val { + t.Errorf("expected %v to be %v, got %v", c.ins, c.val, val) + } + } +} diff --git a/examples/basic_histogram/main.go b/examples/basic_histogram/main.go new file mode 100644 index 0000000..008d7a5 --- /dev/null +++ b/examples/basic_histogram/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "math/rand" + "time" + + "github.com/performancecopilot/speed" +) + +func main() { + max := int64(100) + + c, err := speed.NewPCPClient("histogram_test") + if err != nil { + panic(err) + } + + m, err := speed.NewPCPHistogram("hist", 0, max, 5, "a sample histogram") + if err != nil { + panic(err) + } + + c.MustRegister(m) + + c.MustStart() + defer c.MustStop() + + for i := 0; i < 60; i++ { + v := rand.Int63n(max) + + fmt.Println("recording", v) + m.MustRecord(v) + + time.Sleep(time.Second) + } +} diff --git a/instance.go b/instance.go index 60c055a..1bb497b 100644 --- a/instance.go +++ b/instance.go @@ -3,6 +3,15 @@ package speed // Instances defines a valid collection of instance name and values type Instances map[string]interface{} +// Keys collects and returns all the keys in all instance values +func (i Instances) Keys() []string { + s := make([]string, 0, len(i)) + for k := range i { + s = append(s, k) + } + return s +} + // pcpInstance wraps a PCP compatible Instance type pcpInstance struct { name string diff --git a/metrics.go b/metrics.go index 1a9c920..3ebf240 100644 --- a/metrics.go +++ b/metrics.go @@ -7,6 +7,7 @@ import ( "sync" "time" + histogram "github.com/codahale/hdrhistogram" "github.com/performancecopilot/speed/bytewriter" ) @@ -1075,3 +1076,228 @@ func (g *PCPGaugeVector) Dec(inc float64, instance string) error { return g.Inc( // MustDec panics if Dec fails func (g *PCPGaugeVector) MustDec(inc float64, instance string) { g.MustInc(-inc, instance) } + +/////////////////////////////////////////////////////////////////////////////// + +// Histogram defines a metric that records a distribution of data +type Histogram interface { + Max() int64 // Maximum value recorded so far + Min() int64 // Minimum value recorded so far + + High() int64 // Highest allowed value + Low() int64 // Lowest allowed value + + Record(int64) error // Records a new value + RecordN(int64, int64) error // Records multiple instances of the same value + + MustRecord(int64) + MustRecordN(int64, int64) + + Mean() float64 // Mean of all recorded data + Variance() float64 // Variance of all recorded data + StandardDeviation() float64 // StandardDeviation of all recorded data + Percentile(float64) float64 // Percentile returns the value at the passed percentile +} + +/////////////////////////////////////////////////////////////////////////////// + +// PCPHistogram implements a histogram for PCP backed by the coda hale hdrhistogram +// https://github.com/codahale/hdrhistogram +type PCPHistogram struct { + *pcpInstanceMetric + mutex sync.RWMutex + h *histogram.Histogram +} + +// the maximum and minimum values that can be recorded by a histogram +const ( + HistogramMin = 0 + HistogramMax = 3600000000 +) + +func normalize(low, high int64, sigfigures int) (int64, int64, int) { + if low < HistogramMin { + low = HistogramMin + } + + if low > HistogramMax { + low = HistogramMax + } + + if high < HistogramMin { + high = HistogramMin + } + + if high > HistogramMax { + high = HistogramMax + } + + if sigfigures < 1 { + sigfigures = 1 + } + + if sigfigures > 5 { + sigfigures = 5 + } + + return low, high, sigfigures +} + +// NewPCPHistogram returns a new instance of PCPHistogram +// the lowest value for low is 0 +// the highest value for high is 3,600,000,000 +// the value of sigfigures can be between 1 and 5 +func NewPCPHistogram(name string, low, high int64, sigfigures int, desc ...string) (*PCPHistogram, error) { + if low > high { + return nil, errors.New("low cannot be larger than high") + } + + low, high, sigfigures = normalize(low, high, sigfigures) + + h := histogram.New(low, high, sigfigures) + + instances := []string{"min", "max", "mean", "variance", "standard_deviation"} + vals := make(Instances) + for _, s := range instances { + vals[s] = float64(0) + } + + m, err := generateInstanceMetric(vals, name, instances, DoubleType, InstantSemantics, OneUnit, desc...) + if err != nil { + return nil, err + } + + return &PCPHistogram{m, sync.RWMutex{}, h}, nil +} + +// High returns the maximum recordable value +func (h *PCPHistogram) High() int64 { return h.h.LowestTrackableValue() } + +// Low returns the minimum recordable value +func (h *PCPHistogram) Low() int64 { return h.h.HighestTrackableValue() } + +// Max returns the maximum recorded value so far +func (h *PCPHistogram) Max() int64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + return int64(h.vals["max"].val.(float64)) +} + +// Min returns the minimum recorded value so far +func (h *PCPHistogram) Min() int64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + return int64(h.vals["min"].val.(float64)) +} + +func (h *PCPHistogram) update() error { + updateinstance := func(instance string, val float64) error { + if h.vals[instance].val != val { + return h.setInstance(val, instance) + } + return nil + } + + if err := updateinstance("min", float64(h.h.Min())); err != nil { + return err + } + + if err := updateinstance("max", float64(h.h.Max())); err != nil { + return err + } + + if err := updateinstance("mean", h.h.Mean()); err != nil { + return err + } + + stddev := h.h.StdDev() + + if err := updateinstance("standard_deviation", stddev); err != nil { + return err + } + + if err := updateinstance("variance", stddev*stddev); err != nil { + return err + } + + return nil +} + +// Record records a new value +func (h *PCPHistogram) Record(val int64) error { + h.mutex.Lock() + defer h.mutex.Unlock() + + err := h.h.RecordValue(val) + if err != nil { + return err + } + + return h.update() +} + +// MustRecord panics if Record fails +func (h *PCPHistogram) MustRecord(val int64) { + if err := h.Record(val); err != nil { + panic(err) + } +} + +// RecordN records multiple instances of the same value +func (h *PCPHistogram) RecordN(val, n int64) error { + h.mutex.Lock() + defer h.mutex.Unlock() + + err := h.h.RecordValues(val, n) + if err != nil { + return err + } + + return h.update() +} + +// MustRecordN panics if RecordN fails +func (h *PCPHistogram) MustRecordN(val, n int64) { + if err := h.RecordN(val, n); err != nil { + panic(err) + } +} + +// Mean returns the mean of all values recorded so far +func (h *PCPHistogram) Mean() float64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + return h.vals["mean"].val.(float64) +} + +// StandardDeviation returns the standard deviation of all values recorded so far +func (h *PCPHistogram) StandardDeviation() float64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + return h.vals["standard_deviation"].val.(float64) +} + +// Variance returns the variance of all values recorded so far +func (h *PCPHistogram) Variance() float64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + return h.vals["variance"].val.(float64) +} + +// Percentile returns the value at the passed percentile +func (h *PCPHistogram) Percentile(p float64) int64 { return h.h.ValueAtQuantile(p) } + +// HistogramBucket is a single histogram bucket within a fixed range +type HistogramBucket struct { + From, To, Count int64 +} + +// Buckets returns a list of histogram buckets +func (h *PCPHistogram) Buckets() []*HistogramBucket { + b := h.h.Distribution() + buckets := make([]*HistogramBucket, len(b)) + for i := 0; i < len(b); i++ { + buckets[i] = &HistogramBucket{b[i].From, b[i].To, b[i].Count} + } + return buckets +} diff --git a/vendor/github.com/codahale/hdrhistogram/LICENSE b/vendor/github.com/codahale/hdrhistogram/LICENSE new file mode 100644 index 0000000..f9835c2 --- /dev/null +++ b/vendor/github.com/codahale/hdrhistogram/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Coda Hale + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/codahale/hdrhistogram/README.md b/vendor/github.com/codahale/hdrhistogram/README.md new file mode 100644 index 0000000..614b197 --- /dev/null +++ b/vendor/github.com/codahale/hdrhistogram/README.md @@ -0,0 +1,15 @@ +hdrhistogram +============ + +[![Build Status](https://travis-ci.org/codahale/hdrhistogram.png?branch=master)](https://travis-ci.org/codahale/hdrhistogram) + +A pure Go implementation of the [HDR Histogram](https://github.com/HdrHistogram/HdrHistogram). + +> A Histogram that supports recording and analyzing sampled data value counts +> across a configurable integer value range with configurable value precision +> within the range. Value precision is expressed as the number of significant +> digits in the value recording, and provides control over value quantization +> behavior across the value range and the subsequent value resolution at any +> given level. + +For documentation, check [godoc](http://godoc.org/github.com/codahale/hdrhistogram). diff --git a/vendor/github.com/codahale/hdrhistogram/hdr.go b/vendor/github.com/codahale/hdrhistogram/hdr.go new file mode 100644 index 0000000..02540dd --- /dev/null +++ b/vendor/github.com/codahale/hdrhistogram/hdr.go @@ -0,0 +1,563 @@ +// Package hdrhistogram provides an implementation of Gil Tene's HDR Histogram +// data structure. The HDR Histogram allows for fast and accurate analysis of +// the extreme ranges of data with non-normal distributions, like latency. +package hdrhistogram + +import ( + "fmt" + "math" +) + +// A Bracket is a part of a cumulative distribution. +type Bracket struct { + Quantile float64 + Count, ValueAt int64 +} + +// A Snapshot is an exported view of a Histogram, useful for serializing them. +// A Histogram can be constructed from it by passing it to Import. +type Snapshot struct { + LowestTrackableValue int64 + HighestTrackableValue int64 + SignificantFigures int64 + Counts []int64 +} + +// A Histogram is a lossy data structure used to record the distribution of +// non-normally distributed data (like latency) with a high degree of accuracy +// and a bounded degree of precision. +type Histogram struct { + lowestTrackableValue int64 + highestTrackableValue int64 + unitMagnitude int64 + significantFigures int64 + subBucketHalfCountMagnitude int32 + subBucketHalfCount int32 + subBucketMask int64 + subBucketCount int32 + bucketCount int32 + countsLen int32 + totalCount int64 + counts []int64 +} + +// New returns a new Histogram instance capable of tracking values in the given +// range and with the given amount of precision. +func New(minValue, maxValue int64, sigfigs int) *Histogram { + if sigfigs < 1 || 5 < sigfigs { + panic(fmt.Errorf("sigfigs must be [1,5] (was %d)", sigfigs)) + } + + largestValueWithSingleUnitResolution := 2 * math.Pow10(sigfigs) + subBucketCountMagnitude := int32(math.Ceil(math.Log2(float64(largestValueWithSingleUnitResolution)))) + + subBucketHalfCountMagnitude := subBucketCountMagnitude + if subBucketHalfCountMagnitude < 1 { + subBucketHalfCountMagnitude = 1 + } + subBucketHalfCountMagnitude-- + + unitMagnitude := int32(math.Floor(math.Log2(float64(minValue)))) + if unitMagnitude < 0 { + unitMagnitude = 0 + } + + subBucketCount := int32(math.Pow(2, float64(subBucketHalfCountMagnitude)+1)) + + subBucketHalfCount := subBucketCount / 2 + subBucketMask := int64(subBucketCount-1) << uint(unitMagnitude) + + // determine exponent range needed to support the trackable value with no + // overflow: + smallestUntrackableValue := int64(subBucketCount) << uint(unitMagnitude) + bucketsNeeded := int32(1) + for smallestUntrackableValue < maxValue { + smallestUntrackableValue <<= 1 + bucketsNeeded++ + } + + bucketCount := bucketsNeeded + countsLen := (bucketCount + 1) * (subBucketCount / 2) + + return &Histogram{ + lowestTrackableValue: minValue, + highestTrackableValue: maxValue, + unitMagnitude: int64(unitMagnitude), + significantFigures: int64(sigfigs), + subBucketHalfCountMagnitude: subBucketHalfCountMagnitude, + subBucketHalfCount: subBucketHalfCount, + subBucketMask: subBucketMask, + subBucketCount: subBucketCount, + bucketCount: bucketCount, + countsLen: countsLen, + totalCount: 0, + counts: make([]int64, countsLen), + } +} + +// ByteSize returns an estimate of the amount of memory allocated to the +// histogram in bytes. +// +// N.B.: This does not take into account the overhead for slices, which are +// small, constant, and specific to the compiler version. +func (h *Histogram) ByteSize() int { + return 6*8 + 5*4 + len(h.counts)*8 +} + +// Merge merges the data stored in the given histogram with the receiver, +// returning the number of recorded values which had to be dropped. +func (h *Histogram) Merge(from *Histogram) (dropped int64) { + i := from.rIterator() + for i.next() { + v := i.valueFromIdx + c := i.countAtIdx + + if h.RecordValues(v, c) != nil { + dropped += c + } + } + + return +} + +// TotalCount returns total number of values recorded. +func (h *Histogram) TotalCount() int64 { + return h.totalCount +} + +// Max returns the approximate maximum recorded value. +func (h *Histogram) Max() int64 { + var max int64 + i := h.iterator() + for i.next() { + if i.countAtIdx != 0 { + max = i.highestEquivalentValue + } + } + return h.highestEquivalentValue(max) +} + +// Min returns the approximate minimum recorded value. +func (h *Histogram) Min() int64 { + var min int64 + i := h.iterator() + for i.next() { + if i.countAtIdx != 0 && min == 0 { + min = i.highestEquivalentValue + break + } + } + return h.lowestEquivalentValue(min) +} + +// Mean returns the approximate arithmetic mean of the recorded values. +func (h *Histogram) Mean() float64 { + if h.totalCount == 0 { + return 0 + } + var total int64 + i := h.iterator() + for i.next() { + if i.countAtIdx != 0 { + total += i.countAtIdx * h.medianEquivalentValue(i.valueFromIdx) + } + } + return float64(total) / float64(h.totalCount) +} + +// StdDev returns the approximate standard deviation of the recorded values. +func (h *Histogram) StdDev() float64 { + if h.totalCount == 0 { + return 0 + } + + mean := h.Mean() + geometricDevTotal := 0.0 + + i := h.iterator() + for i.next() { + if i.countAtIdx != 0 { + dev := float64(h.medianEquivalentValue(i.valueFromIdx)) - mean + geometricDevTotal += (dev * dev) * float64(i.countAtIdx) + } + } + + return math.Sqrt(geometricDevTotal / float64(h.totalCount)) +} + +// Reset deletes all recorded values and restores the histogram to its original +// state. +func (h *Histogram) Reset() { + h.totalCount = 0 + for i := range h.counts { + h.counts[i] = 0 + } +} + +// RecordValue records the given value, returning an error if the value is out +// of range. +func (h *Histogram) RecordValue(v int64) error { + return h.RecordValues(v, 1) +} + +// RecordCorrectedValue records the given value, correcting for stalls in the +// recording process. This only works for processes which are recording values +// at an expected interval (e.g., doing jitter analysis). Processes which are +// recording ad-hoc values (e.g., latency for incoming requests) can't take +// advantage of this. +func (h *Histogram) RecordCorrectedValue(v, expectedInterval int64) error { + if err := h.RecordValue(v); err != nil { + return err + } + + if expectedInterval <= 0 || v <= expectedInterval { + return nil + } + + missingValue := v - expectedInterval + for missingValue >= expectedInterval { + if err := h.RecordValue(missingValue); err != nil { + return err + } + missingValue -= expectedInterval + } + + return nil +} + +// RecordValues records n occurrences of the given value, returning an error if +// the value is out of range. +func (h *Histogram) RecordValues(v, n int64) error { + idx := h.countsIndexFor(v) + if idx < 0 || int(h.countsLen) <= idx { + return fmt.Errorf("value %d is too large to be recorded", v) + } + h.counts[idx] += n + h.totalCount += n + + return nil +} + +// ValueAtQuantile returns the recorded value at the given quantile (0..100). +func (h *Histogram) ValueAtQuantile(q float64) int64 { + if q > 100 { + q = 100 + } + + total := int64(0) + countAtPercentile := int64(((q / 100) * float64(h.totalCount)) + 0.5) + + i := h.iterator() + for i.next() { + total += i.countAtIdx + if total >= countAtPercentile { + return h.highestEquivalentValue(i.valueFromIdx) + } + } + + return 0 +} + +// CumulativeDistribution returns an ordered list of brackets of the +// distribution of recorded values. +func (h *Histogram) CumulativeDistribution() []Bracket { + var result []Bracket + + i := h.pIterator(1) + for i.next() { + result = append(result, Bracket{ + Quantile: i.percentile, + Count: i.countToIdx, + ValueAt: i.highestEquivalentValue, + }) + } + + return result +} + +// SignificantFigures returns the significant figures used to create the +// histogram +func (h *Histogram) SignificantFigures() int64 { + return h.significantFigures +} + +// LowestTrackableValue returns the lower bound on values that will be added +// to the histogram +func (h *Histogram) LowestTrackableValue() int64 { + return h.lowestTrackableValue +} + +// HighestTrackableValue returns the upper bound on values that will be added +// to the histogram +func (h *Histogram) HighestTrackableValue() int64 { + return h.highestTrackableValue +} + +// Histogram bar for plotting +type Bar struct { + From, To, Count int64 +} + +// Pretty print as csv for easy plotting +func (b Bar) String() string { + return fmt.Sprintf("%v, %v, %v\n", b.From, b.To, b.Count) +} + +// Distribution returns an ordered list of bars of the +// distribution of recorded values, counts can be normalized to a probability +func (h *Histogram) Distribution() (result []Bar) { + i := h.iterator() + for i.next() { + result = append(result, Bar{ + Count: i.countAtIdx, + From: h.lowestEquivalentValue(i.valueFromIdx), + To: i.highestEquivalentValue, + }) + } + + return result +} + +// Equals returns true if the two Histograms are equivalent, false if not. +func (h *Histogram) Equals(other *Histogram) bool { + switch { + case + h.lowestTrackableValue != other.lowestTrackableValue, + h.highestTrackableValue != other.highestTrackableValue, + h.unitMagnitude != other.unitMagnitude, + h.significantFigures != other.significantFigures, + h.subBucketHalfCountMagnitude != other.subBucketHalfCountMagnitude, + h.subBucketHalfCount != other.subBucketHalfCount, + h.subBucketMask != other.subBucketMask, + h.subBucketCount != other.subBucketCount, + h.bucketCount != other.bucketCount, + h.countsLen != other.countsLen, + h.totalCount != other.totalCount: + return false + default: + for i, c := range h.counts { + if c != other.counts[i] { + return false + } + } + } + return true +} + +// Export returns a snapshot view of the Histogram. This can be later passed to +// Import to construct a new Histogram with the same state. +func (h *Histogram) Export() *Snapshot { + return &Snapshot{ + LowestTrackableValue: h.lowestTrackableValue, + HighestTrackableValue: h.highestTrackableValue, + SignificantFigures: h.significantFigures, + Counts: h.counts, + } +} + +// Import returns a new Histogram populated from the Snapshot data. +func Import(s *Snapshot) *Histogram { + h := New(s.LowestTrackableValue, s.HighestTrackableValue, int(s.SignificantFigures)) + h.counts = s.Counts + totalCount := int64(0) + for i := int32(0); i < h.countsLen; i++ { + countAtIndex := h.counts[i] + if countAtIndex > 0 { + totalCount += countAtIndex + } + } + h.totalCount = totalCount + return h +} + +func (h *Histogram) iterator() *iterator { + return &iterator{ + h: h, + subBucketIdx: -1, + } +} + +func (h *Histogram) rIterator() *rIterator { + return &rIterator{ + iterator: iterator{ + h: h, + subBucketIdx: -1, + }, + } +} + +func (h *Histogram) pIterator(ticksPerHalfDistance int32) *pIterator { + return &pIterator{ + iterator: iterator{ + h: h, + subBucketIdx: -1, + }, + ticksPerHalfDistance: ticksPerHalfDistance, + } +} + +func (h *Histogram) sizeOfEquivalentValueRange(v int64) int64 { + bucketIdx := h.getBucketIndex(v) + subBucketIdx := h.getSubBucketIdx(v, bucketIdx) + adjustedBucket := bucketIdx + if subBucketIdx >= h.subBucketCount { + adjustedBucket++ + } + return int64(1) << uint(h.unitMagnitude+int64(adjustedBucket)) +} + +func (h *Histogram) valueFromIndex(bucketIdx, subBucketIdx int32) int64 { + return int64(subBucketIdx) << uint(int64(bucketIdx)+h.unitMagnitude) +} + +func (h *Histogram) lowestEquivalentValue(v int64) int64 { + bucketIdx := h.getBucketIndex(v) + subBucketIdx := h.getSubBucketIdx(v, bucketIdx) + return h.valueFromIndex(bucketIdx, subBucketIdx) +} + +func (h *Histogram) nextNonEquivalentValue(v int64) int64 { + return h.lowestEquivalentValue(v) + h.sizeOfEquivalentValueRange(v) +} + +func (h *Histogram) highestEquivalentValue(v int64) int64 { + return h.nextNonEquivalentValue(v) - 1 +} + +func (h *Histogram) medianEquivalentValue(v int64) int64 { + return h.lowestEquivalentValue(v) + (h.sizeOfEquivalentValueRange(v) >> 1) +} + +func (h *Histogram) getCountAtIndex(bucketIdx, subBucketIdx int32) int64 { + return h.counts[h.countsIndex(bucketIdx, subBucketIdx)] +} + +func (h *Histogram) countsIndex(bucketIdx, subBucketIdx int32) int32 { + bucketBaseIdx := (bucketIdx + 1) << uint(h.subBucketHalfCountMagnitude) + offsetInBucket := subBucketIdx - h.subBucketHalfCount + return bucketBaseIdx + offsetInBucket +} + +func (h *Histogram) getBucketIndex(v int64) int32 { + pow2Ceiling := bitLen(v | h.subBucketMask) + return int32(pow2Ceiling - int64(h.unitMagnitude) - + int64(h.subBucketHalfCountMagnitude+1)) +} + +func (h *Histogram) getSubBucketIdx(v int64, idx int32) int32 { + return int32(v >> uint(int64(idx)+int64(h.unitMagnitude))) +} + +func (h *Histogram) countsIndexFor(v int64) int { + bucketIdx := h.getBucketIndex(v) + subBucketIdx := h.getSubBucketIdx(v, bucketIdx) + return int(h.countsIndex(bucketIdx, subBucketIdx)) +} + +type iterator struct { + h *Histogram + bucketIdx, subBucketIdx int32 + countAtIdx, countToIdx, valueFromIdx int64 + highestEquivalentValue int64 +} + +func (i *iterator) next() bool { + if i.countToIdx >= i.h.totalCount { + return false + } + + // increment bucket + i.subBucketIdx++ + if i.subBucketIdx >= i.h.subBucketCount { + i.subBucketIdx = i.h.subBucketHalfCount + i.bucketIdx++ + } + + if i.bucketIdx >= i.h.bucketCount { + return false + } + + i.countAtIdx = i.h.getCountAtIndex(i.bucketIdx, i.subBucketIdx) + i.countToIdx += i.countAtIdx + i.valueFromIdx = i.h.valueFromIndex(i.bucketIdx, i.subBucketIdx) + i.highestEquivalentValue = i.h.highestEquivalentValue(i.valueFromIdx) + + return true +} + +type rIterator struct { + iterator + countAddedThisStep int64 +} + +func (r *rIterator) next() bool { + for r.iterator.next() { + if r.countAtIdx != 0 { + r.countAddedThisStep = r.countAtIdx + return true + } + } + return false +} + +type pIterator struct { + iterator + seenLastValue bool + ticksPerHalfDistance int32 + percentileToIteratorTo float64 + percentile float64 +} + +func (p *pIterator) next() bool { + if !(p.countToIdx < p.h.totalCount) { + if p.seenLastValue { + return false + } + + p.seenLastValue = true + p.percentile = 100 + + return true + } + + if p.subBucketIdx == -1 && !p.iterator.next() { + return false + } + + var done = false + for !done { + currentPercentile := (100.0 * float64(p.countToIdx)) / float64(p.h.totalCount) + if p.countAtIdx != 0 && p.percentileToIteratorTo <= currentPercentile { + p.percentile = p.percentileToIteratorTo + halfDistance := math.Trunc(math.Pow(2, math.Trunc(math.Log2(100.0/(100.0-p.percentileToIteratorTo)))+1)) + percentileReportingTicks := float64(p.ticksPerHalfDistance) * halfDistance + p.percentileToIteratorTo += 100.0 / percentileReportingTicks + return true + } + done = !p.iterator.next() + } + + return true +} + +func bitLen(x int64) (n int64) { + for ; x >= 0x8000; x >>= 16 { + n += 16 + } + if x >= 0x80 { + x >>= 8 + n += 8 + } + if x >= 0x8 { + x >>= 4 + n += 4 + } + if x >= 0x2 { + x >>= 2 + n += 2 + } + if x >= 0x1 { + n++ + } + return +} diff --git a/vendor/github.com/codahale/hdrhistogram/window.go b/vendor/github.com/codahale/hdrhistogram/window.go new file mode 100644 index 0000000..dc43612 --- /dev/null +++ b/vendor/github.com/codahale/hdrhistogram/window.go @@ -0,0 +1,45 @@ +package hdrhistogram + +// A WindowedHistogram combines histograms to provide windowed statistics. +type WindowedHistogram struct { + idx int + h []Histogram + m *Histogram + + Current *Histogram +} + +// NewWindowed creates a new WindowedHistogram with N underlying histograms with +// the given parameters. +func NewWindowed(n int, minValue, maxValue int64, sigfigs int) *WindowedHistogram { + w := WindowedHistogram{ + idx: -1, + h: make([]Histogram, n), + m: New(minValue, maxValue, sigfigs), + } + + for i := range w.h { + w.h[i] = *New(minValue, maxValue, sigfigs) + } + w.Rotate() + + return &w +} + +// Merge returns a histogram which includes the recorded values from all the +// sections of the window. +func (w *WindowedHistogram) Merge() *Histogram { + w.m.Reset() + for _, h := range w.h { + w.m.Merge(&h) + } + return w.m +} + +// Rotate resets the oldest histogram and rotates it to be used as the current +// histogram. +func (w *WindowedHistogram) Rotate() { + w.idx++ + w.Current = &w.h[w.idx%len(w.h)] + w.Current.Reset() +} diff --git a/vendor/vendor.json b/vendor/vendor.json index b5230da..cca5642 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -11,6 +11,12 @@ "versionExact": "v0.10.0" }, { + "checksumSHA1": "W7DNxQwdGQ2YO3PR3jARJg6Iqsw=", + "path": "github.com/codahale/hdrhistogram", + "revision": "f8ad88b59a584afeee9d334eff879b104439117b", + "revisionTime": "2016-04-25T23:15:03Z" + }, + { "checksumSHA1": "hBgLmZ/4mCxmnH88mqFKBkpJFUY=", "path": "github.com/mgutz/ansi", "revision": "c286dcecd19ff979eeb73ea444e479b903f2cfcb",