diff --git a/Makefile b/Makefile index f093e0c..9108f41 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ deps: gometalinter --install --update lint: - gometalinter ./... --vendor --deadline=10000s + gometalinter ./... --vendor --deadline=10000s --dupl-threshold=150 test: go test ./... diff --git a/client_test.go b/client_test.go index 535f30a..f33e356 100644 --- a/client_test.go +++ b/client_test.go @@ -4,6 +4,8 @@ import ( "fmt" "os" "testing" + + "github.com/performancecopilot/speed/mmvdump" ) func TestMmvFileLocation(t *testing.T) { @@ -117,3 +119,375 @@ func TestMapping(t *testing.T) { EraseFileOnStop = false } + +func matchMetricDesc(m PCPMetric, metric *mmvdump.Metric, t *testing.T) { + if int32(metric.Sem) != int32(m.Semantics()) { + t.Errorf("expected semantics to be %v, got %v", m.Semantics(), MetricSemantics(metric.Sem)) + } + + if int32(metric.Typ) != int32(m.Type()) { + t.Errorf("expected type to be %v, got %v", m.Type(), MetricType(metric.Typ)) + } + + if int32(metric.Unit) != int32(m.Unit().PMAPI()) { + t.Errorf("expected unit to be %v, got %v", m.Unit(), metric.Unit) + } + + if metric.Shorttext != uint64(m.ShortDescription().offset) { + t.Errorf("expected shorttext to be %v, got %v", m.ShortDescription().offset, metric.Shorttext) + } + + if metric.Longtext != uint64(m.LongDescription().offset) { + t.Errorf("expected longtext to be %v, got %v", m.LongDescription().offset, metric.Longtext) + } +} + +func matchSingletonMetric(m *PCPSingletonMetric, metric *mmvdump.Metric, t *testing.T) { + if metric.Indom != mmvdump.NoIndom { + t.Error("expected indom to be null") + } + + matchMetricDesc(m, metric, t) +} + +func matchSingletonValue(m *PCPSingletonMetric, value *mmvdump.Value, t *testing.T) { + if value.Metric != uint64(m.descoffset) { + t.Errorf("expected value's metric to be at %v", m.descoffset) + } + + if av, err := mmvdump.FixedVal(value.Val, mmvdump.Int32Type); err != nil || av.(int32) != m.val.(int32) { + t.Errorf("expected the value to be %v, got %v", 10, av) + } + + if value.Instance != 0 { + t.Errorf("expected value instance to be 0") + } +} + +func matchString(s *PCPString, str *mmvdump.String, t *testing.T) { + if s == nil { + t.Error("expected PCPString to not be nil") + } + + sv := string(str.Payload[:len(s.val)]) + if sv != s.val { + t.Errorf("expected %v, got %v", s.val, sv) + } +} + +func matchInstanceMetric(m *PCPInstanceMetric, met *mmvdump.Metric, t *testing.T) { + if uint32(met.Indom) != m.indom.id { + t.Errorf("expected indom id to be %d, got %d", m.indom.id, met.Indom) + } + + matchMetricDesc(m, met, t) +} + +func matchInstanceValue(v *mmvdump.Value, i *instanceValue, ins string, met *PCPInstanceMetric, t *testing.T) { + if v.Metric != uint64(met.descoffset) { + t.Errorf("expected value's metric to be at %v", met.descoffset) + } + + if v.Instance == 0 { + t.Errorf("expected instance offset to not be 0") + } + + if met.indom == nil { + t.Errorf("expected indom to be non nil") + } else if in := met.indom.instances[ins]; in == nil { + t.Errorf("expected the instance domain to have an instance %v", ins) + } else if in.offset != int(v.Instance) { + t.Errorf("expected the value's instance to be at offset %v, found at %v", in.offset, v.Instance) + } + + if av, err := mmvdump.FixedVal(v.Val, mmvdump.Uint32Type); err != nil || av.(uint32) != i.val.(uint32) { + t.Errorf("expected the value to be %v, got %v", i.val, av) + } +} + +func matchSingletonMetricAndValue(met *PCPSingletonMetric, metrics map[uint64]*mmvdump.Metric, values map[uint64]*mmvdump.Value, t *testing.T) { + metric, ok := metrics[uint64(met.descoffset)] + if !ok { + t.Errorf("expected a metric at offset %v", met.descoffset) + } else { + matchSingletonMetric(met, metric, t) + } + + mv, ok := values[uint64(met.valueoffset)] + if !ok { + t.Errorf("expected a value at offset %v", met.valueoffset) + } else { + matchSingletonValue(met, mv, t) + } +} + +func matchInstanceMetricAndValues(met *PCPInstanceMetric, metrics map[uint64]*mmvdump.Metric, values map[uint64]*mmvdump.Value, t *testing.T) { + metric, ok := metrics[uint64(met.descoffset)] + if !ok { + t.Errorf("expected a metric at offset %v", met.descoffset) + } else { + matchInstanceMetric(met, metric, t) + } + + for n, i := range met.vals { + mv, ok := values[uint64(i.offset)] + if !ok { + t.Errorf("expected a value at offset %v", i.offset) + } else { + matchInstanceValue(mv, i, n, met, t) + } + } +} + +func matchMetricsAndValues(metrics map[uint64]*mmvdump.Metric, values map[uint64]*mmvdump.Value, c *PCPClient, t *testing.T) { + if c.Registry().MetricCount() != len(metrics) { + t.Errorf("expected %v metrics, got %v", c.Registry().MetricCount(), len(metrics)) + } + + if c.Registry().ValuesCount() != len(values) { + t.Errorf("expected %v values, got %v", c.Registry().ValuesCount(), len(values)) + } + + for _, m := range c.r.metrics { + switch met := m.(type) { + case *PCPSingletonMetric: + matchSingletonMetricAndValue(met, metrics, values, t) + case *PCPInstanceMetric: + matchInstanceMetricAndValues(met, metrics, values, t) + } + } +} + +func TestWritingSingletonMetric(t *testing.T) { + c, err := NewPCPClient("test", ProcessFlag) + if err != nil { + t.Error(err) + return + } + + met, err := NewPCPSingletonMetric(10, "test.1", Int32Type, CounterSemantics, OneUnit, "test", "") + if err != nil { + t.Error(err) + return + } + c.MustRegister(met) + + c.MustStart() + defer c.MustStop() + + h, toc, m, v, i, ind, s, err := mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return + } + + if int(h.Toc) != len(toc) { + t.Errorf("expected the number of tocs specified in the header and the number of tocs in the toc array to be the same, h.Toc = %d, len(tocs) = %d", h.Toc, len(toc)) + } + + if h.Toc != 3 { + t.Errorf("expected client to write %d tocs, written %d", 3, h.Toc) + } + + if h.Flag != int32(ProcessFlag) { + t.Errorf("expected client to write a ProcessFlag, writing %v", MMVFlag(h.Flag)) + } + + matchMetricsAndValues(m, v, c, t) + + // strings + + if len(s) != 1 { + t.Error("expected one string") + } + + matchString(met.shortDescription, s[uint64(met.shortDescription.offset)], t) + + // instances + + if len(i) != 0 { + t.Error("expected no instances when writing a singleton metric") + } + + // indoms + + if len(ind) != 0 { + t.Error("expected no indoms when writing a singleton metric") + } +} + +func matchInstance(i *mmvdump.Instance, pi *pcpInstance, id *PCPInstanceDomain, t *testing.T) { + if i.Indom != uint64(id.offset) { + t.Errorf("expected indom offset to be %d, got %d", i.Indom, id.offset) + } + + if in := i.External[:len(pi.name)]; pi.name != string(in) { + t.Errorf("expected instance name to be %v, got %v", pi.name, in) + } +} + +func matchInstanceDomain(id *mmvdump.InstanceDomain, pid *PCPInstanceDomain, t *testing.T) { + if pid.InstanceCount() != int(id.Count) { + t.Errorf("expected %d instances in instance domain %v, got %d", pid.InstanceCount(), pid.Name(), id.Count) + } + + if id.Offset != uint64(pid.instanceOffset) { + t.Errorf("expected instance offset to be %d, got %d", pid.instanceOffset, id.Offset) + } + + if id.Shorttext != uint64(pid.shortDescription.offset) { + t.Errorf("expected short description to be %d, got %d", pid.shortDescription.offset, id.Shorttext) + } + + if id.Longtext != uint64(pid.longDescription.offset) { + t.Errorf("expected long description to be %d, got %d", pid.longDescription.offset, id.Longtext) + } +} + +func matchInstancesAndInstanceDomains( + ins map[uint64]*mmvdump.Instance, + ids map[uint64]*mmvdump.InstanceDomain, + c *PCPClient, + t *testing.T, +) { + if len(ins) != c.r.InstanceCount() { + t.Errorf("expected %d instances, got %d", c.r.InstanceCount(), len(ins)) + } + + for _, id := range c.r.instanceDomains { + if off := uint64(id.offset); ids[off] == nil { + t.Errorf("expected an instance domain at %d", id.offset) + } else { + matchInstanceDomain(ids[off], id, t) + } + + for _, i := range id.instances { + if ioff := uint64(i.offset); ins[ioff] == nil { + t.Errorf("expected an instance domain at %d", ioff) + } else { + matchInstance(ins[ioff], i, id, t) + } + } + } +} + +func TestWritingInstanceMetric(t *testing.T) { + c, err := NewPCPClient("test", ProcessFlag) + if err != nil { + t.Error(err) + return + } + + id, err := NewPCPInstanceDomain("testid", []string{"a", "b", "c"}, "testid", "") + if err != nil { + t.Error(err) + return + } + c.MustRegisterIndom(id) + + m, err := NewPCPInstanceMetric(Instances{ + "a": 1, + "b": 2, + "c": 3, + }, "test.1", id, Uint32Type, CounterSemantics, OneUnit, "", "test long description") + if err != nil { + t.Error(err) + return + } + + c.MustRegister(m) + + c.MustStart() + defer c.MustStop() + + h, tocs, mets, vals, ins, ids, ss, err := mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return + } + + if int(h.Toc) != len(tocs) { + t.Errorf("expected the number of tocs specified in the header and the number of tocs in the toc array to be the same, h.Toc = %d, len(tocs) = %d", h.Toc, len(tocs)) + } + + if h.Toc != 5 { + t.Errorf("expected client to write %d tocs, written %d", 5, h.Toc) + } + + if h.Flag != int32(ProcessFlag) { + t.Errorf("expected client to write a ProcessFlag, writing %v", MMVFlag(h.Flag)) + } + + matchMetricsAndValues(mets, vals, c, t) + + matchInstancesAndInstanceDomains(ins, ids, c, t) + + // strings + + if off := id.shortDescription.offset; ss[uint64(off)] != nil { + matchString(id.shortDescription, ss[uint64(off)], t) + } else { + t.Errorf("expected a string at offset %v", off) + } + + if off := m.longDescription.offset; ss[uint64(off)] != nil { + matchString(m.longDescription, ss[uint64(off)], t) + } else { + t.Errorf("expected a string at offset %v", off) + } +} + +func TestStringValueWriting(t *testing.T) { + c, err := NewPCPClient("test", ProcessFlag) + if err != nil { + t.Error(err) + return + } + + metric := c.MustRegisterString("test.str", "kirk", CounterSemantics, StringType, OneUnit) + c.MustStart() + defer c.MustStop() + + h, _, _, v, _, _, s, err := mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return + } + + if h.Toc != 3 { + t.Errorf("expected toc to be 3, not %v", h.Toc) + } + + sm := metric.(*PCPSingletonMetric) + + if val, ok := v[uint64(sm.valueoffset)]; !ok { + t.Errorf("expected value at %v", sm.valueoffset) + } else { + add := uint64(val.Extra) + if str, ok := s[add]; !ok { + t.Errorf("expected a string at address %v", add) + } else { + if v := string(str.Payload[:4]); v != "kirk" { + t.Errorf("expected metric value to be kirk, not %v", v) + } + } + } + + sm.Set("spock") + + _, _, _, v, _, _, s, err = mmvdump.Dump(c.buffer.Bytes()) + if err != nil { + t.Error(err) + return + } + + val := v[uint64(sm.valueoffset)] + add := uint64(val.Extra) + if str, ok := s[add]; !ok { + t.Errorf("expected a string at address %v", add) + } else { + if v := string(str.Payload[:5]); v != "spock" { + t.Errorf("expected metric value to be spock, not %v", v) + } + } +} diff --git a/mmvdump/README.md b/mmvdump/README.md new file mode 100644 index 0000000..4b4fb94 --- /dev/null +++ b/mmvdump/README.md @@ -0,0 +1,15 @@ +# mmvdump [![GoDoc](https://godoc.org/github.com/performancecopilot/speed/mmvdump?status.svg)](https://godoc.org/github.com/performancecopilot/speed/mmvdump) + +Package mmvdump implements a go port of the C mmvdump utility included in PCP Core + +https://github.com/performancecopilot/pcp/blob/master/src/pmdas/mmv/mmvdump.c + +It has been written for maximum portability with the C equivalent, without having to use cgo or any other ninja stuff + +the main difference is that the reader is separate from the cli with the reading primarily implemented in mmvdump.go while the cli is implemented in cmd/mmvdump + +the cli application is completely go gettable and outputs the same things, in mostly the same way as the C cli app, to try it out, + +``` +go get github.com/performancecopilot/speed/mmvdump/cmd/mmvdump +``` \ No newline at end of file diff --git a/mmvdump/cmd/mmvdump/main.go b/mmvdump/cmd/mmvdump/main.go new file mode 100644 index 0000000..16b34b0 --- /dev/null +++ b/mmvdump/cmd/mmvdump/main.go @@ -0,0 +1,196 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/performancecopilot/speed/mmvdump" +) + +var ( + header *mmvdump.Header + tocs []*mmvdump.Toc + metrics map[uint64]*mmvdump.Metric + values map[uint64]*mmvdump.Value + instances map[uint64]*mmvdump.Instance + indoms map[uint64]*mmvdump.InstanceDomain + strings map[uint64]*mmvdump.String +) + +func printInstance(offset uint64) { + i := instances[offset] + indom := indoms[i.Indom] + fmt.Printf("\t[%v/%v] instance = [%v/%v]\n", indom.Serial, offset, i.Internal, string(i.External[:])) +} + +func printInstanceDomain(offset uint64) { + indom := indoms[offset] + fmt.Printf("\t[%v/%v] %d instances, starting at offset %d\n", indom.Serial, offset, indom.Count, indom.Offset) + + if indom.Shorttext == 0 { + fmt.Printf("\t\t(no shorttext)\n") + } else { + fmt.Printf("\t\tshorttext=%v\n", string(strings[indom.Shorttext].Payload[:])) + } + + if indom.Longtext == 0 { + fmt.Printf("\t\t(no longtext)\n") + } else { + fmt.Printf("\t\tlongtext=%v\n", string(strings[indom.Longtext].Payload[:])) + } +} + +func printMetric(offset uint64) { + m := metrics[offset] + + fmt.Printf("\t[%v/%v] %v\n", m.Item, offset, string(m.Name[:])) + fmt.Printf("\t\ttype=%v (0x%x), sem=%v (0x%x), pad=0x%x\n", m.Typ, int(m.Typ), m.Sem, int(m.Sem), m.Padding) + fmt.Printf("\t\tunits=%v\n", m.Unit) + + if m.Indom == mmvdump.NoIndom { + fmt.Printf("\t\t(no indom)\n") + } else { + fmt.Printf("\t\tindom=%d\n", m.Indom) + } + + if m.Shorttext == 0 { + fmt.Printf("\t\t(no shorttext)\n") + } else { + fmt.Printf("\t\tshorttext=%v\n", string(strings[m.Shorttext].Payload[:])) + } + + if m.Longtext == 0 { + fmt.Printf("\t\t(no longtext)\n") + } else { + fmt.Printf("\t\tlongtext=%v\n", string(strings[m.Longtext].Payload[:])) + } +} + +func printValue(offset uint64) { + v := values[offset] + m := metrics[v.Metric] + + fmt.Printf("\t[%v/%v] %v", m.Item, offset, string(m.Name[:])) + + var ( + a interface{} + err error + ) + + if m.Typ != mmvdump.StringType { + a, err = mmvdump.FixedVal(v.Val, m.Typ) + } else { + v, ok := strings[uint64(v.Extra)] + if !ok { + panic("invalid string address") + } + a = string(v.Payload[:]) + } + + if m.Indom != mmvdump.NoIndom { + i := instances[v.Instance] + fmt.Printf("[%d or \"%s\"]", i.Internal, string(i.External[:])) + } + + if err != nil { + panic(err) + } + + fmt.Printf(" = %v\n", a) +} + +func printString(offset uint64) { + fmt.Printf("\t[%v] %v\n", offset, string(strings[offset].Payload[:])) +} + +func data(file string) []byte { + f, err := os.Open(file) + if err != nil { + panic(err) + } + + fi, err := os.Stat(file) + if err != nil { + panic(err) + } + + len := fi.Size() + data := make([]byte, len) + + _, err = f.Read(data) + if err != nil { + panic(err) + } + + return data +} + +func main() { + flag.Parse() + + if flag.NArg() < 1 { + fmt.Println("usage: mmvdump ") + return + } + + file := flag.Arg(0) + d := data(file) + + var err error + header, tocs, metrics, values, instances, indoms, strings, err = mmvdump.Dump(d) + if err != nil { + panic(err) + } + + fmt.Printf(` +File = %v +Version = %v +Generated = %v +Toc Count = %v +Cluster = %v +Process = %v +Flags = 0x%x + +`, file, header.Version, header.G1, header.Toc, header.Cluster, header.Process, int(header.Flag)) + + toff := mmvdump.HeaderLength + var ( + itemtype string + itemsize uint64 + printItem func(uint64) + ) + + for ti, toc := range tocs { + switch toc.Type { + case mmvdump.TocInstances: + itemtype = "instances" + itemsize = mmvdump.InstanceLength + printItem = printInstance + case mmvdump.TocIndoms: + itemtype = "indoms" + itemsize = mmvdump.InstanceDomainLength + printItem = printInstanceDomain + case mmvdump.TocMetrics: + itemtype = "metric" + itemsize = mmvdump.MetricLength + printItem = printMetric + case mmvdump.TocValues: + itemtype = "values" + itemsize = mmvdump.ValueLength + printItem = printValue + case mmvdump.TocStrings: + itemtype = "strings" + itemsize = mmvdump.StringLength + printItem = printString + } + + fmt.Printf("TOC[%v], offset: %v, %v offset: %v (%v entries)\n", ti, toff, itemtype, toc.Offset, toc.Count) + for i, offset := int32(0), toc.Offset; i < toc.Count; i, offset = i+1, offset+itemsize { + printItem(offset) + } + fmt.Println() + + toff += mmvdump.TocLength + } +} diff --git a/mmvdump/mmvdump.go b/mmvdump/mmvdump.go new file mode 100644 index 0000000..4a7731c --- /dev/null +++ b/mmvdump/mmvdump.go @@ -0,0 +1,350 @@ +// Package mmvdump implements a go port of the C mmvdump utility included in PCP Core +// +// https://github.com/performancecopilot/pcp/blob/master/src/pmdas/mmv/mmvdump.c +// +// It has been written for maximum portability with the C equivalent, without having to use cgo or any other ninja stuff +// +// the main difference is that the reader is separate from the cli with the reading primarily implemented in mmvdump.go while the cli is implemented in cmd/mmvdump +// +// the cli application is completely go gettable and outputs the same things, in mostly the same way as the C cli app, to try it out, +// +// ``` +// go get github.com/performancecopilot/speed/mmvdump/cmd/mmvdump +// ``` +package mmvdump + +import ( + "errors" + "fmt" + "math" + "sync" + "unsafe" +) + +func readHeader(data []byte) (*Header, error) { + if uint64(len(data)) < HeaderLength { + return nil, errors.New("file too small to contain a valid Header") + } + + header := (*Header)(unsafe.Pointer(&data[0])) + + if m := header.Magic[:3]; string(m) != "MMV" { + return nil, fmt.Errorf("Bad Magic: %v", string(m)) + } + + if header.G1 != header.G2 { + return nil, fmt.Errorf("Mismatched version numbers, %v and %v", header.G1, header.G2) + } + + return header, nil +} + +func readToc(data []byte, offset uint64) (*Toc, error) { + if uint64(len(data)) < offset+TocLength { + return nil, errors.New("Incomplete/Partially Written TOC") + } + + return (*Toc)(unsafe.Pointer(&data[offset])), nil +} + +func readInstance(data []byte, offset uint64) (*Instance, error) { + if uint64(len(data)) < offset+InstanceLength { + return nil, errors.New("Incomplete/Partially Written Instance") + } + + return (*Instance)(unsafe.Pointer(&data[offset])), nil +} + +func readInstanceDomain(data []byte, offset uint64) (*InstanceDomain, error) { + if uint64(len(data)) < offset+InstanceDomainLength { + return nil, errors.New("Incomplete/Partially Written InstanceDomain") + } + + return (*InstanceDomain)(unsafe.Pointer(&data[offset])), nil +} + +func readMetric(data []byte, offset uint64) (*Metric, error) { + if uint64(len(data)) < offset+MetricLength { + return nil, errors.New("Incomplete/Partially Written Metric") + } + + return (*Metric)(unsafe.Pointer(&data[offset])), nil +} + +func readValue(data []byte, offset uint64) (*Value, error) { + if uint64(len(data)) < offset+ValueLength { + return nil, errors.New("Incomplete/Partially Written Value") + } + + return (*Value)(unsafe.Pointer(&data[offset])), nil +} + +func readString(data []byte, offset uint64) (*String, error) { + if uint64(len(data)) < offset+StringLength { + return nil, errors.New("Incomplete/Partially Written String") + } + + return (*String)(unsafe.Pointer(&data[offset])), nil +} + +func readTocs(data []byte, count int32) ([]*Toc, error) { + tocs := make([]*Toc, count) + + for i := int32(0); i < count; i++ { + t, err := readToc(data, HeaderLength+uint64(i)*TocLength) + if err != nil { + return nil, err + } + tocs[i] = t + } + + return tocs, nil +} + +func readInstances(data []byte, offset uint64, count int32) (map[uint64]*Instance, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + + instances := make(map[uint64]*Instance) + + var ( + instance *Instance + err error + m sync.Mutex + ) + + for i := int32(0); i < count; i, offset = i+1, offset+InstanceLength { + go func(offset uint64) { + instance, err = readInstance(data, offset) + if err == nil { + m.Lock() + instances[offset] = instance + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err + } + + return instances, nil +} + +func readInstanceDomains(data []byte, offset uint64, count int32) (map[uint64]*InstanceDomain, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + + indoms := make(map[uint64]*InstanceDomain) + + var ( + indom *InstanceDomain + err error + m sync.Mutex + ) + + for i := int32(0); i < count; i, offset = i+1, offset+InstanceDomainLength { + go func(offset uint64) { + indom, err = readInstanceDomain(data, offset) + if err == nil { + m.Lock() + indoms[offset] = indom + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err + } + + return indoms, nil +} + +func readMetrics(data []byte, offset uint64, count int32) (map[uint64]*Metric, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + + metrics := make(map[uint64]*Metric) + + var ( + metric *Metric + err error + m sync.Mutex + ) + + for i := int32(0); i < count; i, offset = i+1, offset+MetricLength { + go func(offset uint64) { + metric, err = readMetric(data, offset) + if err == nil { + m.Lock() + metrics[offset] = metric + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err + } + + return metrics, nil +} + +func readValues(data []byte, offset uint64, count int32) (map[uint64]*Value, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + + values := make(map[uint64]*Value) + + var ( + value *Value + err error + m sync.Mutex + ) + + for i := int32(0); i < count; i, offset = i+1, offset+ValueLength { + go func(offset uint64) { + value, err = readValue(data, offset) + if err == nil { + m.Lock() + values[offset] = value + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err + } + + return values, nil +} + +func readStrings(data []byte, offset uint64, count int32) (map[uint64]*String, error) { + var wg sync.WaitGroup + wg.Add(int(count)) + + strings := make(map[uint64]*String) + + var ( + str *String + err error + m sync.Mutex + ) + + for i := int32(0); i < count; i, offset = i+1, offset+StringLength { + go func(offset uint64) { + str, err = readString(data, offset) + if err == nil { + m.Lock() + strings[offset] = str + m.Unlock() + } + wg.Done() + }(offset) + } + + wg.Wait() + + if err != nil { + return nil, err + } + + return strings, nil +} + +// Dump creates a data dump from the passed data +func Dump(data []byte) ( + h *Header, + tocs []*Toc, + metrics map[uint64]*Metric, + values map[uint64]*Value, + instances map[uint64]*Instance, + indoms map[uint64]*InstanceDomain, + strings map[uint64]*String, + err error, +) { + h, err = readHeader(data) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + + tocs, err = readTocs(data, h.Toc) + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + + var wg sync.WaitGroup + wg.Add(len(tocs)) + + for _, toc := range tocs { + switch toc.Type { + case TocInstances: + go func(offset uint64, count int32) { + instances, err = readInstances(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) + case TocIndoms: + go func(offset uint64, count int32) { + indoms, err = readInstanceDomains(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) + case TocMetrics: + go func(offset uint64, count int32) { + metrics, err = readMetrics(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) + case TocValues: + go func(offset uint64, count int32) { + values, err = readValues(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) + case TocStrings: + go func(offset uint64, count int32) { + strings, err = readStrings(data, offset, count) + wg.Done() + }(toc.Offset, toc.Count) + } + } + + wg.Wait() + + if err != nil { + return nil, nil, nil, nil, nil, nil, nil, err + } + + return +} + +// FixedVal will infer a fixed size value from the passed data +func FixedVal(data uint64, t Type) (interface{}, error) { + switch t { + case Int32Type: + return int32(data), nil + case Uint32Type: + return uint32(data), nil + case Int64Type: + return int64(data), nil + case Uint64Type: + return data, nil + case FloatType: + return math.Float32frombits(uint32(data)), nil + case DoubleType: + return math.Float64frombits(data), nil + } + + return nil, errors.New("invalid type") +} diff --git a/mmvdump/mmvdump_test.go b/mmvdump/mmvdump_test.go new file mode 100644 index 0000000..90a84e0 --- /dev/null +++ b/mmvdump/mmvdump_test.go @@ -0,0 +1,68 @@ +package mmvdump + +import ( + "os" + "testing" +) + +func data(filename string) []byte { + f, err := os.Open(filename) + if err != nil { + panic(err) + } + + s, err := os.Stat(filename) + if err != nil { + panic(err) + } + + data := make([]byte, s.Size()) + n, err := f.Read(data) + if err != nil { + panic(err) + } + + if int64(n) != s.Size() { + panic("Could not read complete file" + filename + " into memory") + } + + return data +} + +func TestMmvDump1(t *testing.T) { + d := data("testdata/test1.mmv") + + h, tocs, metrics, values, instances, indoms, strings, err := Dump(d) + if err != nil { + t.Error(err) + return + } + + if h.G1 != h.G2 { + t.Error("Invalid Header") + } + + if len(tocs) != 3 { + t.Errorf("expected number of tocs %d, got %d", 3, len(tocs)) + } + + if len(indoms) != 0 { + t.Errorf("expected number of indoms %d, got %d", 0, len(indoms)) + } + + if len(strings) != 2 { + t.Errorf("expected number of strings %d, got %d", 2, len(strings)) + } + + if len(metrics) != 1 { + t.Errorf("expected number of metrics %d, got %d", 1, len(metrics)) + } + + if len(values) != 1 { + t.Errorf("expected number of values %d, got %d", 1, len(values)) + } + + if len(instances) != 0 { + t.Errorf("expected number of instances %d, got %d", 0, len(instances)) + } +} diff --git a/mmvdump/pcp.go b/mmvdump/pcp.go new file mode 100644 index 0000000..a5ea25a --- /dev/null +++ b/mmvdump/pcp.go @@ -0,0 +1,161 @@ +package mmvdump + +// MMVVersion is the current mmv format version +const MMVVersion = 1 + +const ( + // NameMax is the maximum allowed length of a name + NameMax = 64 + + // StringMax is the maximum allowed length of a string + StringMax = 256 + + // NoIndom is a constant used to indicate abscence of an indom from a metric + NoIndom = -1 +) + +// Header describes the data in a MMV header +type Header struct { + Magic [4]byte + Version int32 + G1, G2 uint64 + Toc int32 + Flag int32 + Process, Cluster int32 +} + +// TocType is an enumerated type with different types as values +type TocType int32 + +// Values for TocType +const ( + TocIndoms TocType = iota + 1 + TocInstances + TocMetrics + TocValues + TocStrings +) + +//go:generate stringer --type=TocType + +// Toc defines the contents in a valid TOC +type Toc struct { + Type TocType + Count int32 + Offset uint64 +} + +// Instance defines the contents in a valid instance +type Instance struct { + Indom uint64 + Padding uint32 + Internal int32 + External [NameMax]byte +} + +// InstanceDomain defines the contents in a valid instance domain +type InstanceDomain struct { + Serial, Count uint32 + Offset, Shorttext, Longtext uint64 +} + +// Metric defines the contents in a valid Metric +type Metric struct { + Name [NameMax]byte + Item uint32 + Typ Type + Sem Semantics + Unit Unit + Indom int32 + Padding uint32 + Shorttext, Longtext uint64 +} + +// Value defines the contents in a PCP Value +type Value struct { + // uint64 is a holder type here, while printing it is expected that + // the user will infer the value using the Val functions + Val uint64 + + Extra int64 + Metric uint64 + Instance uint64 +} + +// String wraps the payload for a PCP String +type String struct { + Payload [StringMax]byte +} + +// Type is an enumerated type representing all valid types for a metric +type Type int32 + +// Possible values for a Type +const ( + NoSupportType Type = iota - 1 + Int32Type + Uint32Type + Int64Type + Uint64Type + FloatType + DoubleType + StringType + UnknownType Type = 255 +) + +//go:generate stringer --type=Type + +// Unit is an enumerated type with all possible units as values +type Unit uint32 + +// Values for Space Units +const ( + ByteUnit Unit = 1<<28 | iota<<16 + KilobyteUnit + MegabyteUnit + GigabyteUnit + TerabyteUnit + PetabyteUnit + ExabyteUnit +) + +// Values for Time Units +const ( + NanosecondUnit Unit = 1<<24 | iota<<12 + MicrosecondUnit + MillisecondUnit + SecondUnit + MinuteUnit + HourUnit +) + +// Values for Count Units +const ( + OneUnit Unit = 1<<20 | iota<<8 +) + +//go:generate stringer --type=Unit + +// Semantics represents an enumerated type representing all possible semantics of a metric +type Semantics int32 + +// Values for Semantics +const ( + NoSemantics Semantics = 0 + CounterSemantics Semantics = 1 + InstantSemantics Semantics = 3 + DiscreteSemantics Semantics = 4 +) + +//go:generate stringer -type=Semantics + +// Byte Lengths for Different Components +const ( + HeaderLength uint64 = 40 + TocLength = 16 + MetricLength = 104 + ValueLength = 32 + InstanceLength = 80 + InstanceDomainLength = 32 + StringLength = 256 +) diff --git a/mmvdump/testdata/test1.mmv b/mmvdump/testdata/test1.mmv new file mode 100644 index 0000000..63c0e7c Binary files /dev/null and b/mmvdump/testdata/test1.mmv differ diff --git a/mmvdump/testdata/test2.mmv b/mmvdump/testdata/test2.mmv new file mode 100644 index 0000000..0bfeaf6 Binary files /dev/null and b/mmvdump/testdata/test2.mmv differ diff --git a/mmvdump/testdata/test3.mmv b/mmvdump/testdata/test3.mmv new file mode 100644 index 0000000..25044e8 Binary files /dev/null and b/mmvdump/testdata/test3.mmv differ diff --git a/mmvdump/testdata/test4.mmv b/mmvdump/testdata/test4.mmv new file mode 100644 index 0000000..fdce591 Binary files /dev/null and b/mmvdump/testdata/test4.mmv differ