From 2d8be6875ce3f565ec228eacb797f8349a9a738a Mon Sep 17 00:00:00 2001 From: Suyash Date: Sat, 23 Jul 2016 10:37:51 +0530 Subject: [PATCH 01/17] mmvdump: initial library --- mmvdump/mmvdump.go | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++++ mmvdump/pcp.go | 143 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 mmvdump/mmvdump.go create mode 100644 mmvdump/pcp.go diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go new file mode 100644 index 0000000..6df6c06 --- /dev/null +++ b/mmvdump/mmvdump.go @@ -0,0 +1,152 @@ +// Package mmvdump implements the go port of the mmvdump utility inside pcp core +// written in C +package mmvdump + +import ( + "errors" + "fmt" + "unsafe" +) + +func readHeader(data []byte) (*Header, error) { + if uint64(len(data)) < HeaderLength { + return nil, errors.New("file too small to contain a valid Header") + } + + header := (*Header)(unsafe.Pointer(&data[0])) + + if m := header.Magic[:3]; string(m) != "MMV" { + return nil, fmt.Errorf("Bad Magic: %v", string(m)) + } + + if header.G1 != header.G2 { + return nil, fmt.Errorf("Mismatched version numbers, %v and %v", header.G1, header.G2) + } + + return header, nil +} + +func readToc(data []byte, offset uint64) (*Toc, error) { + if uint64(len(data)) < offset+TocLength { + return nil, errors.New("Incomplete/Partially Written TOC") + } + + return (*Toc)(unsafe.Pointer(&data[offset])), nil +} + +func readInstance(data []byte, offset uint64) (*Instance, error) { + if uint64(len(data)) < offset+InstanceLength { + return nil, errors.New("Incomplete/Partially Written Instance") + } + + return (*Instance)(unsafe.Pointer(&data[offset])), nil +} + +func readInstanceDomain(data []byte, offset uint64) (*InstanceDomain, error) { + if uint64(len(data)) < offset+InstanceDomainLength { + return nil, errors.New("Incomplete/Partially Written InstanceDomain") + } + + return (*InstanceDomain)(unsafe.Pointer(&data[offset])), nil +} + +func readMetric(data []byte, offset uint64) (*Metric, error) { + if uint64(len(data)) < offset+MetricLength { + return nil, errors.New("Incomplete/Partially Written Metric") + } + + return (*Metric)(unsafe.Pointer(&data[offset])), nil +} + +func readValue(data []byte, offset uint64) (*Value, error) { + if uint64(len(data)) < offset+ValueLength { + return nil, errors.New("Incomplete/Partially Written Value") + } + + return (*Value)(unsafe.Pointer(&data[offset])), nil +} + +func readString(data []byte, offset uint64) (*String, error) { + if uint64(len(data)) < offset+StringLength { + return nil, errors.New("Incomplete/Partially Written String") + } + + return (*String)(unsafe.Pointer(&data[offset])), nil +} + +func Dump(data []byte) ( + h *Header, + ts map[int32]*Toc, + metrics map[uint32]*Metric, + values []*Value, + instances map[int32]*Instance, + indoms map[uint32]*InstanceDomain, + strings []*String, + err error, +) { + h, err = readHeader(data) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + + ts = make(map[int32]*Toc) + for i := 0; i < int(h.Toc); i++ { + t, err := readToc(data, HeaderLength+uint64(i)*TocLength) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + ts[int32(t.Type)] = t + } + + for _, toc := range ts { + switch toc.Type { + case TocInstances: + instances = make(map[int32]*Instance) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+InstanceLength { + instance, err := readInstance(data, offset) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + instances[instance.Internal] = instance + } + case TocIndoms: + indoms := make(map[uint32]*InstanceDomain) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+InstanceDomainLength { + indom, err := readInstanceDomain(data, offset) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + indoms[indom.Serial] = indom + } + case TocMetrics: + metrics = make(map[uint32]*Metric) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+MetricLength { + metric, err := readMetric(data, offset) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + metrics[metric.Item] = metric + } + case TocValues: + values = make([]*Value, toc.Count) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+ValueLength { + value, err := readValue(data, offset) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + values[i] = value + } + case TocStrings: + strings = make([]*String, toc.Count) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+StringLength { + str, err := readString(data, offset) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + strings[i] = str + } + } + } + + return +} diff --git a/mmvdump/pcp.go b/mmvdump/pcp.go new file mode 100644 index 0000000..fc25a34 --- /dev/null +++ b/mmvdump/pcp.go @@ -0,0 +1,143 @@ +package mmvdump + +const MMVVersion = 1 + +const NAMEMAX = 256 + +// Header describes the data in a MMV header +type Header struct { + Magic [4]byte + Version int32 + G1, G2 uint64 + Toc int32 + Flag int32 + Cluster, Process int32 +} + +// TocType is an enumerated type with different types as values +type TocType int32 + +const ( + TocIndoms TocType = iota + 1 + TocInstances + TocMetrics + TocValues + TocStrings +) + +//go:generate stringer --type=TocType + +// Toc defines the contents in a valid TOC +type Toc struct { + Type TocType + Count int32 + Offset uint64 +} + +// Instance defines the contents in a valid instance +type Instance struct { + Indom uint64 + Padding uint32 + Internal int32 + External [NAMEMAX]byte +} + +// InstanceDomain defines the contents in a valid instance domain +type InstanceDomain struct { + Serial, Count uint32 + offset, shorttext, longtext uint64 +} + +// Metric defines the contents in a valid Metric +type Metric struct { + Name [NAMEMAX]byte + Item uint32 + Typ Type + Sem Semantics + Unit Unit + Indom int32 + Padding uint32 + Shorttext, Longtext uint64 +} + +// Value defines the contents in a PCP Value +type Value struct { + // uint64 is a holder type here, while printing it is expected that + // the user will infer the value using the Metric Type + Val uint64 + + Extra int64 + Metric uint64 + Instance uint64 +} + +// String wraps the payload for a PCP String +type String struct { + Payload [NAMEMAX]byte +} + +// Type is an enumerated type representing all valid types for a metric +type Type int32 + +// Possible values for a Type +const ( + NoSupportType Type = iota - 1 + Int32Type + Uint32Type + Int64Type + Uint64Type + FloatType + DoubleType + StringType + UnknownType Type = 255 +) + +//go:generate stringer --type=Type + +// Unit is an enumerated type with all possible units as values +type Unit uint32 + +const ( + ByteUnit Unit = 1<<28 | iota<<16 + KilobyteUnit + MegabyteUnit + GigabyteUnit + TerabyteUnit + PetabyteUnit + ExabyteUnit +) + +const ( + NanosecondUnit Unit = 1<<24 | iota<<12 + MicrosecondUnit + MillisecondUnit + SecondUnit + MinuteUnit + HourUnit +) + +const OneUnit Unit = 1<<20 | iota<<8 + +//go:generate stringer --type=Unit + +// Semantics represents an enumerated type representing all possible semantics of a metric +type Semantics int32 + +const ( + NoSemantics Semantics = 0 + CounterSemantics Semantics = 1 + InstantSemantics Semantics = 3 + DiscreteSemantics Semantics = 4 +) + +//go:generate stringer -type=Semantics + +const ( + HeaderLength uint64 = 40 + TocLength = 16 + MetricLength = 104 + ValueLength = 32 + InstanceLength = 80 + InstanceDomainLength = 32 + StringLength = 256 +) From 0d922a0803a0c0954e5b25a7583cd9acc8a741bd Mon Sep 17 00:00:00 2001 From: Suyash Date: Sat, 23 Jul 2016 15:29:46 +0530 Subject: [PATCH 02/17] mmvdump: initial working cli --- mmvdump/cmd/mmvdump/main.go | 149 ++++++++++++++++++++++++++++++++++++++++++++ mmvdump/mmvdump.go | 56 +++++++++++++---- mmvdump/pcp.go | 12 ++-- mmvdump/testdata/test1.mmv | Bin 0 -> 736 bytes 4 files changed, 201 insertions(+), 16 deletions(-) create mode 100644 mmvdump/cmd/mmvdump/main.go create mode 100644 mmvdump/testdata/test1.mmv diff --git a/mmvdump/cmd/mmvdump/main.go b/mmvdump/cmd/mmvdump/main.go new file mode 100644 index 0000000..f95f2fe --- /dev/null +++ b/mmvdump/cmd/mmvdump/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/performancecopilot/speed/mmvdump" +) + +var ( + header *mmvdump.Header + tocs map[int32]*mmvdump.Toc + metrics map[uint64]*mmvdump.Metric + values map[uint64]*mmvdump.Value + instances map[uint64]*mmvdump.Instance + indoms map[uint32]*mmvdump.InstanceDomain + strings map[uint64]*mmvdump.String +) + +func printMetric(offset uint64) { + m := metrics[offset] + + fmt.Printf("\t[%v/%v] %v\n", m.Item, offset, string(m.Name[:])) + fmt.Printf("\t\ttype=%v (0x%x), sem=%v (0x%x), pad=0x%x\n", m.Typ, int(m.Typ), m.Sem, int(m.Sem), m.Padding) + fmt.Printf("\t\tunits=%v\n", m.Unit) + + if m.Indom == mmvdump.NO_INDOM { + fmt.Printf("\t\t(no indom)\n") + } else { + fmt.Printf("\t\tindom=%d\n", m.Indom) + } + + if m.Shorttext == 0 { + fmt.Printf("\t\t(no shorttext)\n") + } else { + fmt.Printf("\t\tshorttext=%v\n", string(strings[m.Shorttext].Payload[:])) + } + + if m.Longtext == 0 { + fmt.Printf("\t\t(no longtext)\n") + } else { + fmt.Printf("\t\tlongtext=%v\n", string(strings[m.Longtext].Payload[:])) + } +} + +func printValue(offset uint64) { + v := values[offset] + m := metrics[v.Metric] + + fmt.Printf("\t[%v/%v] %v", m.Item, offset, string(m.Name[:])) + + var ( + a interface{} + err error + ) + + if m.Typ != mmvdump.StringType { + a, err = mmvdump.FixedVal(v.Val, m.Typ) + } else { + a, err = mmvdump.StringVal(v.Val, strings) + } + + if m.Indom != mmvdump.NO_INDOM { + i := instances[v.Instance] + fmt.Printf("[%d or \"%s\"]", i.Internal, string(i.External[:])) + } + + if err != nil { + panic(err) + } + + fmt.Printf(" = %v\n", a) +} + +func printString(offset uint64) { + fmt.Printf("\t[%v] %v\n", offset, string(strings[offset].Payload[:])) +} + +func main() { + flag.Parse() + + if flag.NArg() < 1 { + panic("Usage: mmvdump ") + } + + file := flag.Arg(0) + + f, err := os.Open(file) + if err != nil { + panic(err) + } + + fi, err := os.Stat(file) + if err != nil { + panic(err) + } + + len := fi.Size() + data := make([]byte, len) + + _, err = f.Read(data) + if err != nil { + panic(err) + } + + header, tocs, metrics, values, instances, indoms, strings, err = mmvdump.Dump(data) + if err != nil { + panic(err) + } + + fmt.Printf(` +File = %v +Version = %v +Generated = %v +Toc Count = %v +Cluster = %v +Process = %v +Flags = 0x%x + +`, file, header.Version, header.G1, header.Toc, header.Cluster, header.Process, int(header.Flag)) + + offset := mmvdump.HeaderLength + + for _, toc := range tocs { + switch toc.Type { + case mmvdump.TocMetrics: + fmt.Printf("TOC[%v], offset: %v, metrics offset: %v (%v entries)\n", int(toc.Type), offset, toc.Offset, toc.Count) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.MetricLength { + printMetric(offset) + } + + case mmvdump.TocValues: + fmt.Printf("TOC[%v], offset: %v, values offset: %v (%v entries)\n", int(toc.Type), offset, toc.Offset, toc.Count) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.ValueLength { + printValue(offset) + } + + case mmvdump.TocStrings: + fmt.Printf("TOC[%v], offset: %v, strings offset: %v (%v entries)\n", int(toc.Type), offset, toc.Offset, toc.Count) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.StringLength { + printString(offset) + } + } + + fmt.Println() + offset += mmvdump.TocLength + } +} diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go index 6df6c06..93219d4 100644 --- a/mmvdump/mmvdump.go +++ b/mmvdump/mmvdump.go @@ -5,6 +5,7 @@ package mmvdump import ( "errors" "fmt" + "math" "unsafe" ) @@ -74,14 +75,15 @@ func readString(data []byte, offset uint64) (*String, error) { return (*String)(unsafe.Pointer(&data[offset])), nil } +// Dump creates a data dump from the passed data func Dump(data []byte) ( h *Header, ts map[int32]*Toc, - metrics map[uint32]*Metric, - values []*Value, - instances map[int32]*Instance, + metrics map[uint64]*Metric, + values map[uint64]*Value, + instances map[uint64]*Instance, indoms map[uint32]*InstanceDomain, - strings []*String, + strings map[uint64]*String, err error, ) { h, err = readHeader(data) @@ -101,13 +103,13 @@ func Dump(data []byte) ( for _, toc := range ts { switch toc.Type { case TocInstances: - instances = make(map[int32]*Instance) + instances = make(map[uint64]*Instance) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+InstanceLength { instance, err := readInstance(data, offset) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } - instances[instance.Internal] = instance + instances[offset] = instance } case TocIndoms: indoms := make(map[uint32]*InstanceDomain) @@ -119,34 +121,64 @@ func Dump(data []byte) ( indoms[indom.Serial] = indom } case TocMetrics: - metrics = make(map[uint32]*Metric) + metrics = make(map[uint64]*Metric) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+MetricLength { metric, err := readMetric(data, offset) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } - metrics[metric.Item] = metric + metrics[offset] = metric } case TocValues: - values = make([]*Value, toc.Count) + values = make(map[uint64]*Value) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+ValueLength { value, err := readValue(data, offset) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } - values[i] = value + values[offset] = value } case TocStrings: - strings = make([]*String, toc.Count) + strings = make(map[uint64]*String) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+StringLength { str, err := readString(data, offset) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } - strings[i] = str + strings[offset] = str } } } return } + +// FixedVal will infer a fixed size value from the passed data +func FixedVal(data uint64, t Type) (interface{}, error) { + switch t { + case Int32Type: + return int32(data), nil + case Uint32Type: + return uint32(data), nil + case Int64Type: + return int64(data), nil + case Uint64Type: + return data, nil + case FloatType: + return math.Float32frombits(uint32(data)), nil + case DoubleType: + return math.Float64frombits(data), nil + } + + return nil, errors.New("invalid type") +} + +// StringVal will infer the string corresponding to an address +func StringVal(data uint64, strings map[uint64]*String) (string, error) { + str, ok := strings[data] + if !ok { + return "", errors.New("invalid string address") + } + + return string(str.Payload[:]), nil +} diff --git a/mmvdump/pcp.go b/mmvdump/pcp.go index fc25a34..224b62c 100644 --- a/mmvdump/pcp.go +++ b/mmvdump/pcp.go @@ -2,7 +2,11 @@ package mmvdump const MMVVersion = 1 -const NAMEMAX = 256 +const ( + NAMEMAX = 64 + STRINGMAX = 256 + NO_INDOM = -1 +) // Header describes the data in a MMV header type Header struct { @@ -11,7 +15,7 @@ type Header struct { G1, G2 uint64 Toc int32 Flag int32 - Cluster, Process int32 + Process, Cluster int32 } // TocType is an enumerated type with different types as values @@ -63,7 +67,7 @@ type Metric struct { // Value defines the contents in a PCP Value type Value struct { // uint64 is a holder type here, while printing it is expected that - // the user will infer the value using the Metric Type + // the user will infer the value using the Val functions Val uint64 Extra int64 @@ -73,7 +77,7 @@ type Value struct { // String wraps the payload for a PCP String type String struct { - Payload [NAMEMAX]byte + Payload [STRINGMAX]byte } // Type is an enumerated type representing all valid types for a metric diff --git a/mmvdump/testdata/test1.mmv b/mmvdump/testdata/test1.mmv new file mode 100644 index 0000000000000000000000000000000000000000..63c0e7ced384e5a5af34e63fc6f222e24db5cb5a GIT binary patch literal 736 zcmebE4P#(rU|@K$qB|T&!D(h7n+b?nN*NgHfi#E@QWpWlAix5|ApQX$1_4$e2C07l z#2`?dnOl&Ps+XK!npcuq1Q8>Jyb3Z52tal)FbFXG2Lcf50f+#C2M`GcEeID*!rTWF za8w8eyF|e^wWKIB874ep=#Y%eVg(>dRDi^Y0yIVxa>3!DP?E2ZlA4>JS6ot*Sdywx Ul98%VT#%ZYqTm?dIUE8B0B>9{;Q#;t literal 0 HcmV?d00001 From af411a3a87410a4695a2d060b92a39b8755c250e Mon Sep 17 00:00:00 2001 From: Suyash Date: Sat, 23 Jul 2016 16:06:17 +0530 Subject: [PATCH 03/17] mmvdump: fix some lint warnings --- mmvdump/cmd/mmvdump/main.go | 45 +++++++++++++++++++++++++-------------------- mmvdump/pcp.go | 25 ++++++++++++++++++------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/mmvdump/cmd/mmvdump/main.go b/mmvdump/cmd/mmvdump/main.go index f95f2fe..27f2020 100644 --- a/mmvdump/cmd/mmvdump/main.go +++ b/mmvdump/cmd/mmvdump/main.go @@ -25,7 +25,7 @@ func printMetric(offset uint64) { fmt.Printf("\t\ttype=%v (0x%x), sem=%v (0x%x), pad=0x%x\n", m.Typ, int(m.Typ), m.Sem, int(m.Sem), m.Padding) fmt.Printf("\t\tunits=%v\n", m.Unit) - if m.Indom == mmvdump.NO_INDOM { + if m.Indom == mmvdump.NoIndom { fmt.Printf("\t\t(no indom)\n") } else { fmt.Printf("\t\tindom=%d\n", m.Indom) @@ -61,7 +61,7 @@ func printValue(offset uint64) { a, err = mmvdump.StringVal(v.Val, strings) } - if m.Indom != mmvdump.NO_INDOM { + if m.Indom != mmvdump.NoIndom { i := instances[v.Instance] fmt.Printf("[%d or \"%s\"]", i.Internal, string(i.External[:])) } @@ -77,23 +77,15 @@ func printString(offset uint64) { fmt.Printf("\t[%v] %v\n", offset, string(strings[offset].Payload[:])) } -func main() { - flag.Parse() - - if flag.NArg() < 1 { - panic("Usage: mmvdump ") - } - - file := flag.Arg(0) - +func data(file string) ([]byte, error) { f, err := os.Open(file) if err != nil { - panic(err) + return nil, err } fi, err := os.Stat(file) if err != nil { - panic(err) + return nil, err } len := fi.Size() @@ -101,10 +93,23 @@ func main() { _, err = f.Read(data) if err != nil { - panic(err) + return nil, err } - header, tocs, metrics, values, instances, indoms, strings, err = mmvdump.Dump(data) + return data, nil +} + +func main() { + flag.Parse() + + if flag.NArg() < 1 { + panic("Usage: mmvdump ") + } + + file := flag.Arg(0) + d, err := data(file) + + header, tocs, metrics, values, instances, indoms, strings, err = mmvdump.Dump(d) if err != nil { panic(err) } @@ -120,30 +125,30 @@ Flags = 0x%x `, file, header.Version, header.G1, header.Toc, header.Cluster, header.Process, int(header.Flag)) - offset := mmvdump.HeaderLength + toff := mmvdump.HeaderLength for _, toc := range tocs { switch toc.Type { case mmvdump.TocMetrics: - fmt.Printf("TOC[%v], offset: %v, metrics offset: %v (%v entries)\n", int(toc.Type), offset, toc.Offset, toc.Count) + fmt.Printf("TOC[%v], offset: %v, metrics offset: %v (%v entries)\n", int(toc.Type), toff, toc.Offset, toc.Count) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.MetricLength { printMetric(offset) } case mmvdump.TocValues: - fmt.Printf("TOC[%v], offset: %v, values offset: %v (%v entries)\n", int(toc.Type), offset, toc.Offset, toc.Count) + fmt.Printf("TOC[%v], offset: %v, values offset: %v (%v entries)\n", int(toc.Type), toff, toc.Offset, toc.Count) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.ValueLength { printValue(offset) } case mmvdump.TocStrings: - fmt.Printf("TOC[%v], offset: %v, strings offset: %v (%v entries)\n", int(toc.Type), offset, toc.Offset, toc.Count) + fmt.Printf("TOC[%v], offset: %v, strings offset: %v (%v entries)\n", int(toc.Type), toff, toc.Offset, toc.Count) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.StringLength { printString(offset) } } fmt.Println() - offset += mmvdump.TocLength + toff += mmvdump.TocLength } } diff --git a/mmvdump/pcp.go b/mmvdump/pcp.go index 224b62c..74ef99f 100644 --- a/mmvdump/pcp.go +++ b/mmvdump/pcp.go @@ -1,11 +1,17 @@ package mmvdump +// MMVVersion is the current mmv format version const MMVVersion = 1 const ( - NAMEMAX = 64 - STRINGMAX = 256 - NO_INDOM = -1 + // Maximum allowed length of a name + NameMax = 64 + + // Maximum allowed length of a string + StringMax = 256 + + // Constant used to indicate abscence of an indom from a metric + NoIndom = -1 ) // Header describes the data in a MMV header @@ -21,6 +27,7 @@ type Header struct { // TocType is an enumerated type with different types as values type TocType int32 +// Values for TocType const ( TocIndoms TocType = iota + 1 TocInstances @@ -43,18 +50,18 @@ type Instance struct { Indom uint64 Padding uint32 Internal int32 - External [NAMEMAX]byte + External [NameMax]byte } // InstanceDomain defines the contents in a valid instance domain type InstanceDomain struct { Serial, Count uint32 - offset, shorttext, longtext uint64 + Offset, Shorttext, Longtext uint64 } // Metric defines the contents in a valid Metric type Metric struct { - Name [NAMEMAX]byte + Name [NameMax]byte Item uint32 Typ Type Sem Semantics @@ -77,7 +84,7 @@ type Value struct { // String wraps the payload for a PCP String type String struct { - Payload [STRINGMAX]byte + Payload [StringMax]byte } // Type is an enumerated type representing all valid types for a metric @@ -101,6 +108,7 @@ const ( // Unit is an enumerated type with all possible units as values type Unit uint32 +// Values for Space Units const ( ByteUnit Unit = 1<<28 | iota<<16 KilobyteUnit @@ -111,6 +119,7 @@ const ( ExabyteUnit ) +// Values for Time Units const ( NanosecondUnit Unit = 1<<24 | iota<<12 MicrosecondUnit @@ -120,6 +129,7 @@ const ( HourUnit ) +// Values for Count Units const OneUnit Unit = 1<<20 | iota<<8 //go:generate stringer --type=Unit @@ -136,6 +146,7 @@ const ( //go:generate stringer -type=Semantics +// Byte Lengths for Different Components const ( HeaderLength uint64 = 40 TocLength = 16 From a2faa89fb568daf540134e38708090dac2fc8267 Mon Sep 17 00:00:00 2001 From: Suyash Date: Sun, 24 Jul 2016 10:08:38 +0530 Subject: [PATCH 04/17] make: increase dupl threshold for lint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f093e0c..eaf281e 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ deps: gometalinter --install --update lint: - gometalinter ./... --vendor --deadline=10000s + gometalinter ./... --vendor --deadline=10000s --dupl-threshold=80 test: go test ./... From 93bfea4a2352cdcc8b8e94bb6f64d3312d3185bf Mon Sep 17 00:00:00 2001 From: Suyash Date: Sun, 24 Jul 2016 10:09:50 +0530 Subject: [PATCH 05/17] mmvdump: make InstanceDomains consistent and make Tocs array --- mmvdump/cmd/mmvdump/main.go | 37 ++++++++++++++++++++++--------------- mmvdump/mmvdump.go | 12 ++++++------ mmvdump/pcp.go | 11 +++++++---- 3 files changed, 35 insertions(+), 25 deletions(-) diff --git a/mmvdump/cmd/mmvdump/main.go b/mmvdump/cmd/mmvdump/main.go index 27f2020..32a8f91 100644 --- a/mmvdump/cmd/mmvdump/main.go +++ b/mmvdump/cmd/mmvdump/main.go @@ -10,11 +10,11 @@ import ( var ( header *mmvdump.Header - tocs map[int32]*mmvdump.Toc + tocs []*mmvdump.Toc metrics map[uint64]*mmvdump.Metric values map[uint64]*mmvdump.Value instances map[uint64]*mmvdump.Instance - indoms map[uint32]*mmvdump.InstanceDomain + indoms map[uint64]*mmvdump.InstanceDomain strings map[uint64]*mmvdump.String ) @@ -126,29 +126,36 @@ Flags = 0x%x `, file, header.Version, header.G1, header.Toc, header.Cluster, header.Process, int(header.Flag)) toff := mmvdump.HeaderLength + var ( + itemtype string + itemsize uint64 + printItem func(uint64) + ) - for _, toc := range tocs { + for ti, toc := range tocs { switch toc.Type { case mmvdump.TocMetrics: - fmt.Printf("TOC[%v], offset: %v, metrics offset: %v (%v entries)\n", int(toc.Type), toff, toc.Offset, toc.Count) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.MetricLength { - printMetric(offset) - } + itemtype = "metric" + itemsize = mmvdump.MetricLength + printItem = printMetric case mmvdump.TocValues: - fmt.Printf("TOC[%v], offset: %v, values offset: %v (%v entries)\n", int(toc.Type), toff, toc.Offset, toc.Count) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.ValueLength { - printValue(offset) - } + itemtype = "values" + itemsize = mmvdump.ValueLength + printItem = printValue case mmvdump.TocStrings: - fmt.Printf("TOC[%v], offset: %v, strings offset: %v (%v entries)\n", int(toc.Type), toff, toc.Offset, toc.Count) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+mmvdump.StringLength { - printString(offset) - } + itemtype = "strings" + itemsize = mmvdump.StringLength + printItem = printString } + fmt.Printf("TOC[%v], offset: %v, %v offset: %v (%v entries)\n", ti, toff, itemtype, toc.Offset, toc.Count) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+itemsize { + printItem(offset) + } fmt.Println() + toff += mmvdump.TocLength } } diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go index 93219d4..16ecb25 100644 --- a/mmvdump/mmvdump.go +++ b/mmvdump/mmvdump.go @@ -78,11 +78,11 @@ func readString(data []byte, offset uint64) (*String, error) { // Dump creates a data dump from the passed data func Dump(data []byte) ( h *Header, - ts map[int32]*Toc, + ts []*Toc, metrics map[uint64]*Metric, values map[uint64]*Value, instances map[uint64]*Instance, - indoms map[uint32]*InstanceDomain, + indoms map[uint64]*InstanceDomain, strings map[uint64]*String, err error, ) { @@ -91,13 +91,13 @@ func Dump(data []byte) ( return nil, nil, nil, nil, nil, nil, nil, err } - ts = make(map[int32]*Toc) + ts = make([]*Toc, h.Toc) for i := 0; i < int(h.Toc); i++ { t, err := readToc(data, HeaderLength+uint64(i)*TocLength) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } - ts[int32(t.Type)] = t + ts[i] = t } for _, toc := range ts { @@ -112,13 +112,13 @@ func Dump(data []byte) ( instances[offset] = instance } case TocIndoms: - indoms := make(map[uint32]*InstanceDomain) + indoms := make(map[uint64]*InstanceDomain) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+InstanceDomainLength { indom, err := readInstanceDomain(data, offset) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err } - indoms[indom.Serial] = indom + indoms[offset] = indom } case TocMetrics: metrics = make(map[uint64]*Metric) diff --git a/mmvdump/pcp.go b/mmvdump/pcp.go index 74ef99f..a5ea25a 100644 --- a/mmvdump/pcp.go +++ b/mmvdump/pcp.go @@ -4,13 +4,13 @@ package mmvdump const MMVVersion = 1 const ( - // Maximum allowed length of a name + // NameMax is the maximum allowed length of a name NameMax = 64 - // Maximum allowed length of a string + // StringMax is the maximum allowed length of a string StringMax = 256 - // Constant used to indicate abscence of an indom from a metric + // NoIndom is a constant used to indicate abscence of an indom from a metric NoIndom = -1 ) @@ -130,13 +130,16 @@ const ( ) // Values for Count Units -const OneUnit Unit = 1<<20 | iota<<8 +const ( + OneUnit Unit = 1<<20 | iota<<8 +) //go:generate stringer --type=Unit // Semantics represents an enumerated type representing all possible semantics of a metric type Semantics int32 +// Values for Semantics const ( NoSemantics Semantics = 0 CounterSemantics Semantics = 1 From 4bf59f2b353f4dabb9becd8df9723806fd4fce02 Mon Sep 17 00:00:00 2001 From: Suyash Date: Sun, 24 Jul 2016 10:38:47 +0530 Subject: [PATCH 06/17] mmvdump: add support for printing instances and indoms --- mmvdump/cmd/mmvdump/main.go | 36 ++++++++++++++++++++++++++++++++++-- mmvdump/mmvdump.go | 2 +- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/mmvdump/cmd/mmvdump/main.go b/mmvdump/cmd/mmvdump/main.go index 32a8f91..d43bf9e 100644 --- a/mmvdump/cmd/mmvdump/main.go +++ b/mmvdump/cmd/mmvdump/main.go @@ -18,6 +18,29 @@ var ( strings map[uint64]*mmvdump.String ) +func printInstance(offset uint64) { + i := instances[offset] + indom := indoms[i.Indom] + fmt.Printf("\t[%v/%v] instance = [%v/%v]\n", indom.Serial, offset, i.Internal, string(i.External[:])) +} + +func printInstanceDomain(offset uint64) { + indom := indoms[offset] + fmt.Printf("\t[%v/%v] %d instances, starting at offset %d\n", indom.Serial, offset, indom.Count, indom.Offset) + + if indom.Shorttext == 0 { + fmt.Printf("\t\t(no shorttext)\n") + } else { + fmt.Printf("\t\tshorttext=%v\n", string(strings[indom.Shorttext].Payload[:])) + } + + if indom.Longtext == 0 { + fmt.Printf("\t\t(no longtext)\n") + } else { + fmt.Printf("\t\tlongtext=%v\n", string(strings[indom.Longtext].Payload[:])) + } +} + func printMetric(offset uint64) { m := metrics[offset] @@ -108,6 +131,9 @@ func main() { file := flag.Arg(0) d, err := data(file) + if err != nil { + panic(err) + } header, tocs, metrics, values, instances, indoms, strings, err = mmvdump.Dump(d) if err != nil { @@ -134,16 +160,22 @@ Flags = 0x%x for ti, toc := range tocs { switch toc.Type { + case mmvdump.TocInstances: + itemtype = "instances" + itemsize = mmvdump.InstanceLength + printItem = printInstance + case mmvdump.TocIndoms: + itemtype = "indoms" + itemsize = mmvdump.InstanceDomainLength + printItem = printInstanceDomain case mmvdump.TocMetrics: itemtype = "metric" itemsize = mmvdump.MetricLength printItem = printMetric - case mmvdump.TocValues: itemtype = "values" itemsize = mmvdump.ValueLength printItem = printValue - case mmvdump.TocStrings: itemtype = "strings" itemsize = mmvdump.StringLength diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go index 16ecb25..0927e87 100644 --- a/mmvdump/mmvdump.go +++ b/mmvdump/mmvdump.go @@ -112,7 +112,7 @@ func Dump(data []byte) ( instances[offset] = instance } case TocIndoms: - indoms := make(map[uint64]*InstanceDomain) + indoms = make(map[uint64]*InstanceDomain) for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+InstanceDomainLength { indom, err := readInstanceDomain(data, offset) if err != nil { From 3a6bb89e33cbc9884412b09e7ab21389f95d045d Mon Sep 17 00:00:00 2001 From: Suyash Date: Sun, 24 Jul 2016 11:24:52 +0530 Subject: [PATCH 07/17] mmvdump: refactor --- mmvdump/cmd/mmvdump/main.go | 19 +++--- mmvdump/mmvdump.go | 141 +++++++++++++++++++++++++++++--------------- 2 files changed, 101 insertions(+), 59 deletions(-) diff --git a/mmvdump/cmd/mmvdump/main.go b/mmvdump/cmd/mmvdump/main.go index d43bf9e..2b67e3d 100644 --- a/mmvdump/cmd/mmvdump/main.go +++ b/mmvdump/cmd/mmvdump/main.go @@ -100,15 +100,15 @@ func printString(offset uint64) { fmt.Printf("\t[%v] %v\n", offset, string(strings[offset].Payload[:])) } -func data(file string) ([]byte, error) { +func data(file string) []byte { f, err := os.Open(file) if err != nil { - return nil, err + panic(err) } fi, err := os.Stat(file) if err != nil { - return nil, err + panic(err) } len := fi.Size() @@ -116,25 +116,24 @@ func data(file string) ([]byte, error) { _, err = f.Read(data) if err != nil { - return nil, err + panic(err) } - return data, nil + return data } func main() { flag.Parse() if flag.NArg() < 1 { - panic("Usage: mmvdump ") + fmt.Println("usage: mmvdump ") + return } file := flag.Arg(0) - d, err := data(file) - if err != nil { - panic(err) - } + d := data(file) + var err error header, tocs, metrics, values, instances, indoms, strings, err = mmvdump.Dump(d) if err != nil { panic(err) diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go index 0927e87..116c21c 100644 --- a/mmvdump/mmvdump.go +++ b/mmvdump/mmvdump.go @@ -75,10 +75,89 @@ func readString(data []byte, offset uint64) (*String, error) { return (*String)(unsafe.Pointer(&data[offset])), nil } +func readTocs(data []byte, count int32) ([]*Toc, error) { + tocs := make([]*Toc, count) + + for i := int32(0); i < count; i++ { + t, err := readToc(data, HeaderLength+uint64(i)*TocLength) + if err != nil { + return nil, err + } + tocs[i] = t + } + + return tocs, nil +} + +func readInstances(data []byte, offset uint64, count int32) (map[uint64]*Instance, error) { + instances := make(map[uint64]*Instance) + for i := int32(0); i < count; i, offset = i+1, offset+InstanceLength { + instance, err := readInstance(data, offset) + if err != nil { + return nil, err + } + instances[offset] = instance + } + + return instances, nil +} + +func readInstanceDomains(data []byte, offset uint64, count int32) (map[uint64]*InstanceDomain, error) { + indoms := make(map[uint64]*InstanceDomain) + for i := int32(0); i < count; i, offset = i+1, offset+InstanceDomainLength { + indom, err := readInstanceDomain(data, offset) + if err != nil { + return nil, err + } + indoms[offset] = indom + } + + return indoms, nil +} + +func readMetrics(data []byte, offset uint64, count int32) (map[uint64]*Metric, error) { + metrics := make(map[uint64]*Metric) + for i := int32(0); i < count; i, offset = i+1, offset+MetricLength { + metric, err := readMetric(data, offset) + if err != nil { + return nil, err + } + metrics[offset] = metric + } + + return metrics, nil +} + +func readValues(data []byte, offset uint64, count int32) (map[uint64]*Value, error) { + values := make(map[uint64]*Value) + for i := int32(0); i < count; i, offset = i+1, offset+ValueLength { + value, err := readValue(data, offset) + if err != nil { + return nil, err + } + values[offset] = value + } + + return values, nil +} + +func readStrings(data []byte, offset uint64, count int32) (map[uint64]*String, error) { + strings := make(map[uint64]*String) + for i := int32(0); i < count; i, offset = i+1, offset+StringLength { + str, err := readString(data, offset) + if err != nil { + return nil, err + } + strings[offset] = str + } + + return strings, nil +} + // Dump creates a data dump from the passed data func Dump(data []byte) ( h *Header, - ts []*Toc, + tocs []*Toc, metrics map[uint64]*Metric, values map[uint64]*Value, instances map[uint64]*Instance, @@ -91,62 +170,26 @@ func Dump(data []byte) ( return nil, nil, nil, nil, nil, nil, nil, err } - ts = make([]*Toc, h.Toc) - for i := 0; i < int(h.Toc); i++ { - t, err := readToc(data, HeaderLength+uint64(i)*TocLength) - if err != nil { - return nil, nil, nil, nil, nil, nil, nil, err - } - ts[i] = t + tocs, err = readTocs(data, h.Toc) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err } - for _, toc := range ts { + for _, toc := range tocs { switch toc.Type { case TocInstances: - instances = make(map[uint64]*Instance) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+InstanceLength { - instance, err := readInstance(data, offset) - if err != nil { - return nil, nil, nil, nil, nil, nil, nil, err - } - instances[offset] = instance - } + instances, err = readInstances(data, toc.Offset, toc.Count) case TocIndoms: - indoms = make(map[uint64]*InstanceDomain) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+InstanceDomainLength { - indom, err := readInstanceDomain(data, offset) - if err != nil { - return nil, nil, nil, nil, nil, nil, nil, err - } - indoms[offset] = indom - } + indoms, err = readInstanceDomains(data, toc.Offset, toc.Count) case TocMetrics: - metrics = make(map[uint64]*Metric) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+MetricLength { - metric, err := readMetric(data, offset) - if err != nil { - return nil, nil, nil, nil, nil, nil, nil, err - } - metrics[offset] = metric - } + metrics, err = readMetrics(data, toc.Offset, toc.Count) case TocValues: - values = make(map[uint64]*Value) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+ValueLength { - value, err := readValue(data, offset) - if err != nil { - return nil, nil, nil, nil, nil, nil, nil, err - } - values[offset] = value - } + values, err = readValues(data, toc.Offset, toc.Count) case TocStrings: - strings = make(map[uint64]*String) - for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+StringLength { - str, err := readString(data, offset) - if err != nil { - return nil, nil, nil, nil, nil, nil, nil, err - } - strings[offset] = str - } + strings, err = readStrings(data, toc.Offset, toc.Count) + } + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err } } From 0807418053e9207ba9f417bbf77b59eaeea69e31 Mon Sep 17 00:00:00 2001 From: Suyash Date: Mon, 25 Jul 2016 09:29:22 +0530 Subject: [PATCH 08/17] mmvdump: add a basic test --- mmvdump/mmvdump_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 mmvdump/mmvdump_test.go diff --git a/mmvdump/mmvdump_test.go b/mmvdump/mmvdump_test.go new file mode 100644 index 0000000..55b50a1 --- /dev/null +++ b/mmvdump/mmvdump_test.go @@ -0,0 +1,55 @@ +package mmvdump + +import ( + "os" + "testing" +) + +func TestMmvDump1(t *testing.T) { + f, err := os.Open("testdata/test1.mmv") + if err != nil { + panic(err) + } + + s, err := os.Stat("testdata/test1.mmv") + if err != nil { + panic(err) + } + + data := make([]byte, s.Size()) + f.Read(data) + + h, tocs, metrics, values, instances, indoms, strings, err := Dump(data) + if err != nil { + t.Error(err) + return + } + + if h.G1 != h.G2 { + t.Error("Invalid Header") + } + + if len(tocs) != 3 { + t.Errorf("expected number of tocs %d, got %d", 3, len(tocs)) + } + + if len(indoms) != 0 { + t.Errorf("expected number of indoms %d, got %d", 0, len(indoms)) + } + + if len(strings) != 2 { + t.Errorf("expected number of strings %d, got %d", 2, len(strings)) + } + + if len(metrics) != 1 { + t.Errorf("expected number of strings %d, got %d", 1, len(metrics)) + } + + if len(values) != 1 { + t.Errorf("expected number of strings %d, got %d", 1, len(values)) + } + + if len(instances) != 0 { + t.Errorf("expected number of strings %d, got %d", 0, len(instances)) + } +} From 50662755431dd1cc7b974dd39b610fd55f8d8c1f Mon Sep 17 00:00:00 2001 From: Suyash Date: Mon, 25 Jul 2016 10:24:09 +0530 Subject: [PATCH 09/17] client: initial mmvdump backed tests for testing singleton metric writing --- client_test.go | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/client_test.go b/client_test.go index 535f30a..0308741 100644 --- a/client_test.go +++ b/client_test.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "testing" + + "github.com/performancecopilot/speed/mmvdump" ) func TestMmvFileLocation(t *testing.T) { @@ -117,3 +119,122 @@ func TestMapping(t *testing.T) { EraseFileOnStop = false } + +func TestWritingSingletonMetric(t *testing.T) { + c, err := NewPCPClient("test", ProcessFlag) + if err != nil { + t.Error(err) + return + } + + met, err := NewPCPSingletonMetric(10, "test.1", Int32Type, CounterSemantics, OneUnit, "test", "") + if err != nil { + t.Error(err) + return + } + c.MustRegister(met) + + c.MustStart() + defer c.MustStop() + + h, toc, m, v, i, ind, s, err := mmvdump.Dump(c.buffer.Bytes()) + + if int(h.Toc) != len(toc) { + t.Errorf("expected the number of tocs specified in the header and the number of tocs in the toc array to be the same, h.Toc = %d, len(tocs) = %d", h.Toc, len(toc)) + } + + if h.Toc != 3 { + t.Errorf("expected client to write %d tocs, written %d", 3, h.Toc) + } + + if h.Flag != int32(ProcessFlag) { + t.Errorf("expected client to write a ProcessFlag, writing %v", MMVFlag(h.Flag)) + } + + if len(m) != 1 { + t.Errorf("expected to write %d metrics, writing %d", 1, len(m)) + } + + // metrics + + off := c.r.metricsoffset + metric, ok := m[uint64(off)] + if !ok { + t.Errorf("expected the metric to exist at offset %v", off) + } + + if metric.Indom != mmvdump.NoIndom { + t.Error("expected indom to be null") + } + + if int32(metric.Sem) != int32(CounterSemantics) { + t.Errorf("expected semantics to be %v, got %v", CounterSemantics, MetricSemantics(metric.Sem)) + } + + if int32(metric.Typ) != int32(Int32Type) { + t.Errorf("expected type to be %v, got %v", Int32Type, MetricType(metric.Typ)) + } + + if int32(metric.Unit) != int32(OneUnit) { + t.Errorf("expected unit to be %v, got %v", OneUnit, metric.Unit) + } + + if metric.Shorttext == 0 { + t.Error("expected shorttext offset to not be 0") + } + + if metric.Shorttext != uint64(c.r.stringsoffset) { + t.Errorf("expected shorttext offset to be %v", c.r.stringsoffset) + } + + if metric.Longtext != 0 { + t.Errorf("expected longtext offset to be 0") + } + + // values + + mv, ok := v[uint64(c.r.valuesoffset)] + if !ok { + t.Errorf("expected a value to be written at offset %v", c.r.valuesoffset) + } + + if mv.Metric != uint64(off) { + t.Errorf("expected value's metric to be at %v", off) + } + + if av, err := mmvdump.FixedVal(mv.Val, mmvdump.Int32Type); err != nil || av.(int32) != 10 { + t.Errorf("expected the value to be %v, got %v", 10, av) + } + + if mv.Instance != 0 { + t.Errorf("expected value instance to be 0") + } + + // strings + + if len(s) != 1 { + t.Error("expected one string") + } + + str, ok := s[uint64(c.r.stringsoffset)] + if !ok { + t.Errorf("expected a string to be written at offset %v", c.r.stringsoffset) + } + + sv := string(str.Payload[:4]) + if sv != "test" { + t.Errorf("expected payload to be %v, got %v", "test", sv) + } + + // instances + + if len(i) != 0 { + t.Error("expected no instances when writing a singleton metric") + } + + // indoms + + if len(ind) != 0 { + t.Error("expected no indoms when writing a singleton metric") + } +} From 046d3d3ee7dab6b2742d7c59eb82b40aab437582 Mon Sep 17 00:00:00 2001 From: Suyash Date: Tue, 26 Jul 2016 23:22:23 +0530 Subject: [PATCH 10/17] client: add test for testing writing instance metrics --- client_test.go | 233 +++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 186 insertions(+), 47 deletions(-) diff --git a/client_test.go b/client_test.go index 0308741..75aaddd 100644 --- a/client_test.go +++ b/client_test.go @@ -120,6 +120,85 @@ func TestMapping(t *testing.T) { EraseFileOnStop = false } +func matchSingletonMetric(m *PCPSingletonMetric, metric *mmvdump.Metric, t *testing.T) { + if metric.Indom != mmvdump.NoIndom { + t.Error("expected indom to be null") + } + + if int32(metric.Sem) != int32(m.sem) { + t.Errorf("expected semantics to be %v, got %v", m.sem, MetricSemantics(metric.Sem)) + } + + if int32(metric.Typ) != int32(m.t) { + t.Errorf("expected type to be %v, got %v", m.t, MetricType(metric.Typ)) + } + + if int32(metric.Unit) != int32(m.u.PMAPI()) { + t.Errorf("expected unit to be %v, got %v", m.u, metric.Unit) + } + + if metric.Shorttext != uint64(m.shortDescription.offset) { + t.Errorf("expected shorttext to be %v, got %v", m.shortDescription.offset, metric.Shorttext) + } + + if metric.Longtext != uint64(m.longDescription.offset) { + t.Errorf("expected longtext to be %v, got %v", m.longDescription.offset, metric.Longtext) + } +} + +func matchSingletonValue(m *PCPSingletonMetric, value *mmvdump.Value, t *testing.T) { + if value.Metric != uint64(m.descoffset) { + t.Errorf("expected value's metric to be at %v", m.descoffset) + } + + if av, err := mmvdump.FixedVal(value.Val, mmvdump.Int32Type); err != nil || av.(int32) != m.val.(int32) { + t.Errorf("expected the value to be %v, got %v", 10, av) + } + + if value.Instance != 0 { + t.Errorf("expected value instance to be 0") + } +} + +func matchString(s *PCPString, str *mmvdump.String, t *testing.T) { + if s == nil { + t.Error("expected PCPString to not be nil") + } + sv := string(str.Payload[:len(s.val)]) + if sv != s.val { + t.Errorf("expected %v, got %v", s.val, sv) + } +} + +func matchMetricsAndValues(metrics map[uint64]*mmvdump.Metric, values map[uint64]*mmvdump.Value, c *PCPClient, t *testing.T) { + if c.Registry().MetricCount() != len(metrics) { + t.Errorf("expected %v metrics, got %v", c.Registry().MetricCount(), len(metrics)) + } + + if c.Registry().ValuesCount() != len(values) { + t.Errorf("expected %v values, got %v", c.Registry().ValuesCount(), len(values)) + } + + for _, m := range c.r.metrics { + switch met := m.(type) { + case *PCPSingletonMetric: + metric, ok := metrics[uint64(met.descoffset)] + if !ok { + t.Errorf("expected a metric at offset %v", met.descoffset) + continue + } + matchSingletonMetric(met, metric, t) + + mv, ok := values[uint64(met.valueoffset)] + if !ok { + t.Errorf("expected a value at offset %v", met.valueoffset) + continue + } + matchSingletonValue(met, mv, t) + } + } +} + func TestWritingSingletonMetric(t *testing.T) { c, err := NewPCPClient("test", ProcessFlag) if err != nil { @@ -138,6 +217,10 @@ func TestWritingSingletonMetric(t *testing.T) { defer c.MustStop() h, toc, m, v, i, ind, s, err := mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return + } if int(h.Toc) != len(toc) { t.Errorf("expected the number of tocs specified in the header and the number of tocs in the toc array to be the same, h.Toc = %d, len(tocs) = %d", h.Toc, len(toc)) @@ -151,90 +234,146 @@ func TestWritingSingletonMetric(t *testing.T) { t.Errorf("expected client to write a ProcessFlag, writing %v", MMVFlag(h.Flag)) } - if len(m) != 1 { - t.Errorf("expected to write %d metrics, writing %d", 1, len(m)) + matchMetricsAndValues(m, v, c, t) + + // strings + + if len(s) != 1 { + t.Error("expected one string") } - // metrics + matchString(met.shortDescription, s[uint64(met.shortDescription.offset)], t) - off := c.r.metricsoffset - metric, ok := m[uint64(off)] - if !ok { - t.Errorf("expected the metric to exist at offset %v", off) + // instances + + if len(i) != 0 { + t.Error("expected no instances when writing a singleton metric") } - if metric.Indom != mmvdump.NoIndom { - t.Error("expected indom to be null") + // indoms + + if len(ind) != 0 { + t.Error("expected no indoms when writing a singleton metric") } +} - if int32(metric.Sem) != int32(CounterSemantics) { - t.Errorf("expected semantics to be %v, got %v", CounterSemantics, MetricSemantics(metric.Sem)) +func matchInstance(i *mmvdump.Instance, pi *pcpInstance, id *PCPInstanceDomain, t *testing.T) { + if i.Indom != uint64(id.offset) { + t.Errorf("expected indom offset to be %d, got %d", i.Indom, id.offset) } - if int32(metric.Typ) != int32(Int32Type) { - t.Errorf("expected type to be %v, got %v", Int32Type, MetricType(metric.Typ)) + if in := i.External[:len(pi.name)]; pi.name != string(in) { + t.Errorf("expected instance name to be %v, got %v", pi.name, in) } +} - if int32(metric.Unit) != int32(OneUnit) { - t.Errorf("expected unit to be %v, got %v", OneUnit, metric.Unit) +func matchInstanceDomain(id *mmvdump.InstanceDomain, pid *PCPInstanceDomain, t *testing.T) { + if pid.InstanceCount() != int(id.Count) { + t.Errorf("expected %d instances in instance domain %v, got %d", pid.InstanceCount(), pid.Name(), id.Count) } - if metric.Shorttext == 0 { - t.Error("expected shorttext offset to not be 0") + if id.Offset != uint64(pid.instanceOffset) { + t.Errorf("expected instance offset to be %d, got %d", pid.instanceOffset, id.Offset) } - if metric.Shorttext != uint64(c.r.stringsoffset) { - t.Errorf("expected shorttext offset to be %v", c.r.stringsoffset) + if id.Shorttext != uint64(pid.shortDescription.offset) { + t.Errorf("expected short description to be %d, got %d", pid.shortDescription.offset, id.Shorttext) } - if metric.Longtext != 0 { - t.Errorf("expected longtext offset to be 0") + if id.Longtext != uint64(pid.longDescription.offset) { + t.Errorf("expected long description to be %d, got %d", pid.longDescription.offset, id.Longtext) } +} - // values +func matchInstancesAndInstanceDomains( + ins map[uint64]*mmvdump.Instance, + ids map[uint64]*mmvdump.InstanceDomain, + c *PCPClient, + t *testing.T, +) { + if len(ins) != c.r.InstanceCount() { + t.Errorf("expected %d instances, got %d", c.r.InstanceCount(), len(ins)) + } + + for _, id := range c.r.instanceDomains { + if off := uint64(id.offset); ids[off] == nil { + t.Errorf("expected an instance domain at %d", id.offset) + } else { + matchInstanceDomain(ids[off], id, t) + } - mv, ok := v[uint64(c.r.valuesoffset)] - if !ok { - t.Errorf("expected a value to be written at offset %v", c.r.valuesoffset) + for _, i := range id.instances { + if ioff := uint64(i.offset); ins[ioff] == nil { + t.Errorf("expected an instance domain at %d", ioff) + } else { + matchInstance(ins[ioff], i, id, t) + } + } } +} - if mv.Metric != uint64(off) { - t.Errorf("expected value's metric to be at %v", off) +func TestWritingInstanceMetric(t *testing.T) { + c, err := NewPCPClient("test", ProcessFlag) + if err != nil { + t.Error(err) + return } - if av, err := mmvdump.FixedVal(mv.Val, mmvdump.Int32Type); err != nil || av.(int32) != 10 { - t.Errorf("expected the value to be %v, got %v", 10, av) + id, err := NewPCPInstanceDomain("testid", []string{"a", "b", "c"}, "testid", "") + if err != nil { + t.Error(err) + return } + c.MustRegisterIndom(id) - if mv.Instance != 0 { - t.Errorf("expected value instance to be 0") + m, err := NewPCPInstanceMetric(Instances{ + "a": 1, + "b": 2, + "c": 3, + }, "test.1", id, Uint32Type, CounterSemantics, OneUnit, "", "test long description") + if err != nil { + t.Error(err) + return } - // strings + c.MustRegister(m) - if len(s) != 1 { - t.Error("expected one string") - } + c.MustStart() + defer c.MustStop() - str, ok := s[uint64(c.r.stringsoffset)] - if !ok { - t.Errorf("expected a string to be written at offset %v", c.r.stringsoffset) + h, tocs, mets, vals, ins, ids, ss, err := mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return } - sv := string(str.Payload[:4]) - if sv != "test" { - t.Errorf("expected payload to be %v, got %v", "test", sv) + if int(h.Toc) != len(tocs) { + t.Errorf("expected the number of tocs specified in the header and the number of tocs in the toc array to be the same, h.Toc = %d, len(tocs) = %d", h.Toc, len(tocs)) } - // instances + if h.Toc != 5 { + t.Errorf("expected client to write %d tocs, written %d", 5, h.Toc) + } - if len(i) != 0 { - t.Error("expected no instances when writing a singleton metric") + if h.Flag != int32(ProcessFlag) { + t.Errorf("expected client to write a ProcessFlag, writing %v", MMVFlag(h.Flag)) } - // indoms + matchMetricsAndValues(mets, vals, c, t) - if len(ind) != 0 { - t.Error("expected no indoms when writing a singleton metric") + matchInstancesAndInstanceDomains(ins, ids, c, t) + + // strings + + if off := id.shortDescription.offset; ss[uint64(off)] != nil { + matchString(id.shortDescription, ss[uint64(off)], t) + } else { + t.Errorf("expected a string at offset %v", off) + } + + if off := m.longDescription.offset; ss[uint64(off)] != nil { + matchString(m.longDescription, ss[uint64(off)], t) + } else { + t.Errorf("expected a string at offset %v", off) } } From 1823704901773fd4a382c3cd8e6fb21902fca7bb Mon Sep 17 00:00:00 2001 From: Suyash Date: Tue, 26 Jul 2016 23:39:44 +0530 Subject: [PATCH 11/17] mmvdump: refactor data fetching --- mmvdump/mmvdump_test.go | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/mmvdump/mmvdump_test.go b/mmvdump/mmvdump_test.go index 55b50a1..67c62b6 100644 --- a/mmvdump/mmvdump_test.go +++ b/mmvdump/mmvdump_test.go @@ -5,21 +5,34 @@ import ( "testing" ) -func TestMmvDump1(t *testing.T) { - f, err := os.Open("testdata/test1.mmv") +func data(filename string) []byte { + f, err := os.Open(filename) if err != nil { panic(err) } - s, err := os.Stat("testdata/test1.mmv") + s, err := os.Stat(filename) if err != nil { panic(err) } data := make([]byte, s.Size()) - f.Read(data) + n, err := f.Read(data) + if err != nil { + panic(err) + } + + if int64(n) != s.Size() { + panic("Could not read complete file" + filename + " into memory") + } + + return data +} + +func TestMmvDump1(t *testing.T) { + d := data("testdata/test1.mmv") - h, tocs, metrics, values, instances, indoms, strings, err := Dump(data) + h, tocs, metrics, values, instances, indoms, strings, err := Dump(d) if err != nil { t.Error(err) return From b7cc971c15b2f2775839a98f7535f1bb3366b2c4 Mon Sep 17 00:00:00 2001 From: Suyash Date: Tue, 26 Jul 2016 23:48:45 +0530 Subject: [PATCH 12/17] mmvdump: add package documentation --- mmvdump/README.md | 15 +++++++++++++++ mmvdump/mmvdump.go | 15 +++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 mmvdump/README.md diff --git a/mmvdump/README.md b/mmvdump/README.md new file mode 100644 index 0000000..4b4fb94 --- /dev/null +++ b/mmvdump/README.md @@ -0,0 +1,15 @@ +# mmvdump [![GoDoc](https://godoc.org/github.com/performancecopilot/speed/mmvdump?status.svg)](https://godoc.org/github.com/performancecopilot/speed/mmvdump) + +Package mmvdump implements a go port of the C mmvdump utility included in PCP Core + +https://github.com/performancecopilot/pcp/blob/master/src/pmdas/mmv/mmvdump.c + +It has been written for maximum portability with the C equivalent, without having to use cgo or any other ninja stuff + +the main difference is that the reader is separate from the cli with the reading primarily implemented in mmvdump.go while the cli is implemented in cmd/mmvdump + +the cli application is completely go gettable and outputs the same things, in mostly the same way as the C cli app, to try it out, + +``` +go get github.com/performancecopilot/speed/mmvdump/cmd/mmvdump +``` \ No newline at end of file diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go index 116c21c..3afb1c2 100644 --- a/mmvdump/mmvdump.go +++ b/mmvdump/mmvdump.go @@ -1,5 +1,16 @@ -// Package mmvdump implements the go port of the mmvdump utility inside pcp core -// written in C +// Package mmvdump implements a go port of the C mmvdump utility included in PCP Core +// +// https://github.com/performancecopilot/pcp/blob/master/src/pmdas/mmv/mmvdump.c +// +// It has been written for maximum portability with the C equivalent, without having to use cgo or any other ninja stuff +// +// the main difference is that the reader is separate from the cli with the reading primarily implemented in mmvdump.go while the cli is implemented in cmd/mmvdump +// +// the cli application is completely go gettable and outputs the same things, in mostly the same way as the C cli app, to try it out, +// +// ``` +// go get github.com/performancecopilot/speed/mmvdump/cmd/mmvdump +// ``` package mmvdump import ( From a1be304e4f06dd2c555b87d08e927cc9d73e3344 Mon Sep 17 00:00:00 2001 From: Suyash Date: Wed, 27 Jul 2016 00:36:49 +0530 Subject: [PATCH 13/17] client: test instance metrics and values --- client_test.go | 111 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 85 insertions(+), 26 deletions(-) diff --git a/client_test.go b/client_test.go index 75aaddd..ad01c1c 100644 --- a/client_test.go +++ b/client_test.go @@ -120,30 +120,34 @@ func TestMapping(t *testing.T) { EraseFileOnStop = false } -func matchSingletonMetric(m *PCPSingletonMetric, metric *mmvdump.Metric, t *testing.T) { - if metric.Indom != mmvdump.NoIndom { - t.Error("expected indom to be null") +func matchMetricDesc(m PCPMetric, metric *mmvdump.Metric, t *testing.T) { + if int32(metric.Sem) != int32(m.Semantics()) { + t.Errorf("expected semantics to be %v, got %v", m.Semantics(), MetricSemantics(metric.Sem)) } - if int32(metric.Sem) != int32(m.sem) { - t.Errorf("expected semantics to be %v, got %v", m.sem, MetricSemantics(metric.Sem)) + if int32(metric.Typ) != int32(m.Type()) { + t.Errorf("expected type to be %v, got %v", m.Type(), MetricType(metric.Typ)) } - if int32(metric.Typ) != int32(m.t) { - t.Errorf("expected type to be %v, got %v", m.t, MetricType(metric.Typ)) + if int32(metric.Unit) != int32(m.Unit().PMAPI()) { + t.Errorf("expected unit to be %v, got %v", m.Unit(), metric.Unit) } - if int32(metric.Unit) != int32(m.u.PMAPI()) { - t.Errorf("expected unit to be %v, got %v", m.u, metric.Unit) + if metric.Shorttext != uint64(m.ShortDescription().offset) { + t.Errorf("expected shorttext to be %v, got %v", m.ShortDescription().offset, metric.Shorttext) } - if metric.Shorttext != uint64(m.shortDescription.offset) { - t.Errorf("expected shorttext to be %v, got %v", m.shortDescription.offset, metric.Shorttext) + if metric.Longtext != uint64(m.LongDescription().offset) { + t.Errorf("expected longtext to be %v, got %v", m.LongDescription().offset, metric.Longtext) } +} - if metric.Longtext != uint64(m.longDescription.offset) { - t.Errorf("expected longtext to be %v, got %v", m.longDescription.offset, metric.Longtext) +func matchSingletonMetric(m *PCPSingletonMetric, metric *mmvdump.Metric, t *testing.T) { + if metric.Indom != mmvdump.NoIndom { + t.Error("expected indom to be null") } + + matchMetricDesc(m, metric, t) } func matchSingletonValue(m *PCPSingletonMetric, value *mmvdump.Value, t *testing.T) { @@ -164,12 +168,77 @@ func matchString(s *PCPString, str *mmvdump.String, t *testing.T) { if s == nil { t.Error("expected PCPString to not be nil") } + sv := string(str.Payload[:len(s.val)]) if sv != s.val { t.Errorf("expected %v, got %v", s.val, sv) } } +func matchInstanceMetric(m *PCPInstanceMetric, met *mmvdump.Metric, t *testing.T) { + if uint32(met.Indom) != m.indom.id { + t.Errorf("expected indom id to be %d, got %d", m.indom.id, met.Indom) + } + + matchMetricDesc(m, met, t) +} + +func matchInstanceValue(v *mmvdump.Value, i *instanceValue, ins string, met *PCPInstanceMetric, t *testing.T) { + if v.Metric != uint64(met.descoffset) { + t.Errorf("expected value's metric to be at %v", met.descoffset) + } + + if v.Instance == 0 { + t.Errorf("expected instance offset to not be 0") + } + + if met.indom == nil { + t.Errorf("expected indom to be non nil") + } else if in := met.indom.instances[ins]; in == nil { + t.Errorf("expected the instance domain to have an instance %v", ins) + } else if in.offset != int(v.Instance) { + t.Errorf("expected the value's instance to be at offset %v, found at %v", in.offset, v.Instance) + } + + if av, err := mmvdump.FixedVal(v.Val, mmvdump.Uint32Type); err != nil || av.(uint32) != i.val.(uint32) { + t.Errorf("expected the value to be %v, got %v", i.val, av) + } +} + +func matchSingletonMetricAndValue(met *PCPSingletonMetric, metrics map[uint64]*mmvdump.Metric, values map[uint64]*mmvdump.Value, t *testing.T) { + metric, ok := metrics[uint64(met.descoffset)] + if !ok { + t.Errorf("expected a metric at offset %v", met.descoffset) + } else { + matchSingletonMetric(met, metric, t) + } + + mv, ok := values[uint64(met.valueoffset)] + if !ok { + t.Errorf("expected a value at offset %v", met.valueoffset) + } else { + matchSingletonValue(met, mv, t) + } +} + +func matchInstanceMetricAndValues(met *PCPInstanceMetric, metrics map[uint64]*mmvdump.Metric, values map[uint64]*mmvdump.Value, t *testing.T) { + metric, ok := metrics[uint64(met.descoffset)] + if !ok { + t.Errorf("expected a metric at offset %v", met.descoffset) + } else { + matchInstanceMetric(met, metric, t) + } + + for n, i := range met.vals { + mv, ok := values[uint64(i.offset)] + if !ok { + t.Errorf("expected a value at offset %v", i.offset) + } else { + matchInstanceValue(mv, i, n, met, t) + } + } +} + func matchMetricsAndValues(metrics map[uint64]*mmvdump.Metric, values map[uint64]*mmvdump.Value, c *PCPClient, t *testing.T) { if c.Registry().MetricCount() != len(metrics) { t.Errorf("expected %v metrics, got %v", c.Registry().MetricCount(), len(metrics)) @@ -182,19 +251,9 @@ func matchMetricsAndValues(metrics map[uint64]*mmvdump.Metric, values map[uint64 for _, m := range c.r.metrics { switch met := m.(type) { case *PCPSingletonMetric: - metric, ok := metrics[uint64(met.descoffset)] - if !ok { - t.Errorf("expected a metric at offset %v", met.descoffset) - continue - } - matchSingletonMetric(met, metric, t) - - mv, ok := values[uint64(met.valueoffset)] - if !ok { - t.Errorf("expected a value at offset %v", met.valueoffset) - continue - } - matchSingletonValue(met, mv, t) + matchSingletonMetricAndValue(met, metrics, values, t) + case *PCPInstanceMetric: + matchInstanceMetricAndValues(met, metrics, values, t) } } } From 9625c4d48556aef080d24a1947e2424e9eb94698 Mon Sep 17 00:00:00 2001 From: Suyash Date: Wed, 27 Jul 2016 09:12:50 +0530 Subject: [PATCH 14/17] client: add test for writing string values --- client_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/client_test.go b/client_test.go index ad01c1c..f33e356 100644 --- a/client_test.go +++ b/client_test.go @@ -436,3 +436,58 @@ func TestWritingInstanceMetric(t *testing.T) { t.Errorf("expected a string at offset %v", off) } } + +func TestStringValueWriting(t *testing.T) { + c, err := NewPCPClient("test", ProcessFlag) + if err != nil { + t.Error(err) + return + } + + metric := c.MustRegisterString("test.str", "kirk", CounterSemantics, StringType, OneUnit) + c.MustStart() + defer c.MustStop() + + h, _, _, v, _, _, s, err := mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return + } + + if h.Toc != 3 { + t.Errorf("expected toc to be 3, not %v", h.Toc) + } + + sm := metric.(*PCPSingletonMetric) + + if val, ok := v[uint64(sm.valueoffset)]; !ok { + t.Errorf("expected value at %v", sm.valueoffset) + } else { + add := uint64(val.Extra) + if str, ok := s[add]; !ok { + t.Errorf("expected a string at address %v", add) + } else { + if v := string(str.Payload[:4]); v != "kirk" { + t.Errorf("expected metric value to be kirk, not %v", v) + } + } + } + + sm.Set("spock") + + _, _, _, v, _, _, s, err = mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return + } + + val := v[uint64(sm.valueoffset)] + add := uint64(val.Extra) + if str, ok := s[add]; !ok { + t.Errorf("expected a string at address %v", add) + } else { + if v := string(str.Payload[:5]); v != "spock" { + t.Errorf("expected metric value to be spock, not %v", v) + } + } +} From 5f4d62d0232a525fa683d666b0433f3cd51e2b7d Mon Sep 17 00:00:00 2001 From: Suyash Date: Wed, 27 Jul 2016 09:13:23 +0530 Subject: [PATCH 15/17] mmvdump: remove StringVal --- mmvdump/cmd/mmvdump/main.go | 6 +++++- mmvdump/mmvdump.go | 10 ---------- mmvdump/testdata/test2.mmv | Bin 0 -> 576 bytes mmvdump/testdata/test3.mmv | Bin 0 -> 480 bytes mmvdump/testdata/test4.mmv | Bin 0 -> 480 bytes 5 files changed, 5 insertions(+), 11 deletions(-) create mode 100644 mmvdump/testdata/test2.mmv create mode 100644 mmvdump/testdata/test3.mmv create mode 100644 mmvdump/testdata/test4.mmv diff --git a/mmvdump/cmd/mmvdump/main.go b/mmvdump/cmd/mmvdump/main.go index 2b67e3d..16b34b0 100644 --- a/mmvdump/cmd/mmvdump/main.go +++ b/mmvdump/cmd/mmvdump/main.go @@ -81,7 +81,11 @@ func printValue(offset uint64) { if m.Typ != mmvdump.StringType { a, err = mmvdump.FixedVal(v.Val, m.Typ) } else { - a, err = mmvdump.StringVal(v.Val, strings) + v, ok := strings[uint64(v.Extra)] + if !ok { + panic("invalid string address") + } + a = string(v.Payload[:]) } if m.Indom != mmvdump.NoIndom { diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go index 3afb1c2..ed74b75 100644 --- a/mmvdump/mmvdump.go +++ b/mmvdump/mmvdump.go @@ -226,13 +226,3 @@ func FixedVal(data uint64, t Type) (interface{}, error) { return nil, errors.New("invalid type") } - -// StringVal will infer the string corresponding to an address -func StringVal(data uint64, strings map[uint64]*String) (string, error) { - str, ok := strings[data] - if !ok { - return "", errors.New("invalid string address") - } - - return string(str.Payload[:]), nil -} diff --git a/mmvdump/testdata/test2.mmv b/mmvdump/testdata/test2.mmv new file mode 100644 index 0000000000000000000000000000000000000000..0bfeaf60b702cc7059a23e6e47cdaba0ff19ea76 GIT binary patch literal 576 zcmebE4P#(rU|=}rHYFTL!D$vCn+b@yBpDb4SsB1;KE2Rr(hgxUev1U=Uycg)er{fFcZw b9}Q@{ctL5HBA7fZUKL>yP#PwG1Ih;gD@P!y literal 0 HcmV?d00001 diff --git a/mmvdump/testdata/test3.mmv b/mmvdump/testdata/test3.mmv new file mode 100644 index 0000000000000000000000000000000000000000..25044e84940a5655e432debe35f25c9cb023314a GIT binary patch literal 480 zcmebE4P#(rU|>i(Iz1dn!D(h7n+b>?C^9ho0Wv^*kh%yU1_2fz2JsI7F$k~%F^K;F zh(RDJu|zK~F*mgs!X=sf2egn4s0(B-1A_p=e;@!e(FmB}1E@I6eK2J~`AM01FwO|1 E0Sh=4=Kufz literal 0 HcmV?d00001 diff --git a/mmvdump/testdata/test4.mmv b/mmvdump/testdata/test4.mmv new file mode 100644 index 0000000000000000000000000000000000000000..fdce59140e80c00b9353bc9f8beb25ce1de5ce02 GIT binary patch literal 480 zcmebE4P#(rU|^8eo)HeD;50Lk%>=|QK+!)y28a(*7Xic|zyib|{sABc0ahRe@gD#& x2qYzz=;bBmrWQlEB$NMu7P0|#f$U{q5McNZ1Yjl_0TX-x6^FSGP1%U$0st_65yt=k literal 0 HcmV?d00001 From 31ad2c99d4042789e4e1ab489986769faeb76858 Mon Sep 17 00:00:00 2001 From: Suyash Date: Wed, 27 Jul 2016 10:04:03 +0530 Subject: [PATCH 16/17] make: increase dupl threshold to 150 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eaf281e..9108f41 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ deps: gometalinter --install --update lint: - gometalinter ./... --vendor --deadline=10000s --dupl-threshold=80 + gometalinter ./... --vendor --deadline=10000s --dupl-threshold=150 test: go test ./... From 0be976ea0d8b3bbeec4e125813b5a5efaa6df91e Mon Sep 17 00:00:00 2001 From: Suyash Date: Wed, 27 Jul 2016 10:06:10 +0530 Subject: [PATCH 17/17] mmvdump: make reader concurrent --- mmvdump/mmvdump.go | 188 +++++++++++++++++++++++++++++++++++++++--------- mmvdump/mmvdump_test.go | 6 +- 2 files changed, 158 insertions(+), 36 deletions(-) diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go index ed74b75..4a7731c 100644 --- a/mmvdump/mmvdump.go +++ b/mmvdump/mmvdump.go @@ -17,6 +17,7 @@ import ( "errors" "fmt" "math" + "sync" "unsafe" ) @@ -101,65 +102,165 @@ func readTocs(data []byte, count int32) ([]*Toc, error) { } func readInstances(data []byte, offset uint64, count int32) (map[uint64]*Instance, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + instances := make(map[uint64]*Instance) + + var ( + instance *Instance + err error + m sync.Mutex + ) + for i := int32(0); i < count; i, offset = i+1, offset+InstanceLength { - instance, err := readInstance(data, offset) - if err != nil { - return nil, err - } - instances[offset] = instance + go func(offset uint64) { + instance, err = readInstance(data, offset) + if err == nil { + m.Lock() + instances[offset] = instance + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err } return instances, nil } func readInstanceDomains(data []byte, offset uint64, count int32) (map[uint64]*InstanceDomain, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + indoms := make(map[uint64]*InstanceDomain) + + var ( + indom *InstanceDomain + err error + m sync.Mutex + ) + for i := int32(0); i < count; i, offset = i+1, offset+InstanceDomainLength { - indom, err := readInstanceDomain(data, offset) - if err != nil { - return nil, err - } - indoms[offset] = indom + go func(offset uint64) { + indom, err = readInstanceDomain(data, offset) + if err == nil { + m.Lock() + indoms[offset] = indom + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err } return indoms, nil } func readMetrics(data []byte, offset uint64, count int32) (map[uint64]*Metric, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + metrics := make(map[uint64]*Metric) + + var ( + metric *Metric + err error + m sync.Mutex + ) + for i := int32(0); i < count; i, offset = i+1, offset+MetricLength { - metric, err := readMetric(data, offset) - if err != nil { - return nil, err - } - metrics[offset] = metric + go func(offset uint64) { + metric, err = readMetric(data, offset) + if err == nil { + m.Lock() + metrics[offset] = metric + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err } return metrics, nil } func readValues(data []byte, offset uint64, count int32) (map[uint64]*Value, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + values := make(map[uint64]*Value) + + var ( + value *Value + err error + m sync.Mutex + ) + for i := int32(0); i < count; i, offset = i+1, offset+ValueLength { - value, err := readValue(data, offset) - if err != nil { - return nil, err - } - values[offset] = value + go func(offset uint64) { + value, err = readValue(data, offset) + if err == nil { + m.Lock() + values[offset] = value + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err } return values, nil } func readStrings(data []byte, offset uint64, count int32) (map[uint64]*String, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + strings := make(map[uint64]*String) + + var ( + str *String + err error + m sync.Mutex + ) + for i := int32(0); i < count; i, offset = i+1, offset+StringLength { - str, err := readString(data, offset) - if err != nil { - return nil, err - } - strings[offset] = str + go func(offset uint64) { + str, err = readString(data, offset) + if err == nil { + m.Lock() + strings[offset] = str + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err } return strings, nil @@ -186,24 +287,45 @@ func Dump(data []byte) ( return nil, nil, nil, nil, nil, nil, nil, err } + var wg sync.WaitGroup + wg.Add(len(tocs)) + for _, toc := range tocs { switch toc.Type { case TocInstances: - instances, err = readInstances(data, toc.Offset, toc.Count) + go func(offset uint64, count int32) { + instances, err = readInstances(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) case TocIndoms: - indoms, err = readInstanceDomains(data, toc.Offset, toc.Count) + go func(offset uint64, count int32) { + indoms, err = readInstanceDomains(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) case TocMetrics: - metrics, err = readMetrics(data, toc.Offset, toc.Count) + go func(offset uint64, count int32) { + metrics, err = readMetrics(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) case TocValues: - values, err = readValues(data, toc.Offset, toc.Count) + go func(offset uint64, count int32) { + values, err = readValues(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) case TocStrings: - strings, err = readStrings(data, toc.Offset, toc.Count) - } - if err != nil { - return nil, nil, nil, nil, nil, nil, nil, err + go func(offset uint64, count int32) { + strings, err = readStrings(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) } } + wg.Wait() + + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + return } diff --git a/mmvdump/mmvdump_test.go b/mmvdump/mmvdump_test.go index 67c62b6..90a84e0 100644 --- a/mmvdump/mmvdump_test.go +++ b/mmvdump/mmvdump_test.go @@ -55,14 +55,14 @@ func TestMmvDump1(t *testing.T) { } if len(metrics) != 1 { - t.Errorf("expected number of strings %d, got %d", 1, len(metrics)) + t.Errorf("expected number of metrics %d, got %d", 1, len(metrics)) } if len(values) != 1 { - t.Errorf("expected number of strings %d, got %d", 1, len(values)) + t.Errorf("expected number of values %d, got %d", 1, len(values)) } if len(instances) != 0 { - t.Errorf("expected number of strings %d, got %d", 0, len(instances)) + t.Errorf("expected number of instances %d, got %d", 0, len(instances)) } }