add runc metric collector

connector/runc.go

@@ -75,12 +75,6 @@ func NewRunc() *Runc {
 	}
 
 	go cm.Loop()
-	go func() {
-		time.Sleep(1 * time.Second)
-		for _, c := range cm.containers {
-			cm.needsRefresh <- c.Id
-		}
-	}()
 
 	return cm
 }
@@ -99,11 +93,9 @@ func (cm *Runc) inspect(id string) libcontainer.Container {
 	return libc
 }
 
-func (cm *Runc) refresh(c *container.Container) {
+// update a ctop container from libcontainer
-	libc := cm.inspect(c.Id)
+func (cm *Runc) refresh(libc libcontainer.Container, c *container.Container) {
-	if libc == nil {
+	log.Debugf("refreshing container: %s", c.Id)
-		return
-	}
 
 	status, err := libc.Status()
 	if err != nil {
@@ -129,37 +121,45 @@ func (cm *Runc) refreshAll() {
 
 	for _, i := range list {
 		if i.IsDir() {
+			name := i.Name()
 			// attempt to load
-			libc, err := cm.factory.Load(i.Name())
+			libc := cm.inspect(name)
-			if err != nil {
+			if libc == nil {
-				log.Warningf("failed to read container: %s\n", err)
 				continue
 			}
 
-			c := cm.MustGet(libc.ID())
+			c := cm.MustGet(libc)
-			c.SetMeta("name", i.Name())
+			cm.refresh(libc, c)
-
-			cm.needsRefresh <- c.Id
 		}
 	}
 }
 
 func (cm *Runc) Loop() {
-	for id := range cm.needsRefresh {
+	for {
-		c := cm.MustGet(id)
+		cm.refreshAll()
-		cm.refresh(c)
+		time.Sleep(5 * time.Second)
 	}
 }
 
-// Get a single container in the map, creating one anew if not existing
+// Get a single ctop container in the map matching libc container, creating one anew if not existing
-func (cm *Runc) MustGet(id string) *container.Container {
+func (cm *Runc) MustGet(libc libcontainer.Container) *container.Container {
+	id := libc.ID()
 	c, ok := cm.Get(id)
-	// append container struct for new containers
 	if !ok {
 		// create collector
-		collector := metrics.NewRunc(2)
+		collector := metrics.NewRunc(libc)
+
 		// create container
 		c = container.New(id, collector)
+
+		name := libc.ID()
+		// set initial metadata
+		if len(name) > 12 {
+			name = name[0:12]
+		}
+		c.SetMeta("name", name)
+
+		// add to map
 		cm.lock.Lock()
 		cm.containers[id] = c
 		cm.lock.Unlock()

metrics/runc.go

@@ -1,25 +1,32 @@
 package metrics
 
 import (
-	"math/rand"
 	"time"
+
+	"github.com/opencontainers/runc/libcontainer"
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 )
 
 // Runc collector
 type Runc struct {
 	Metrics
+	id         string
+	libc       libcontainer.Container
 	stream     chan Metrics
 	done       bool
 	running    bool
-	aggression int64
+	interval   int // collection interval, in seconds
+	lastCpu    float64
+	lastSysCpu float64
 }
 
-func NewRunc(a int64) *Runc {
+func NewRunc(libc libcontainer.Container) *Runc {
 	c := &Runc{
 		Metrics:  Metrics{},
-		aggression: a,
+		id:       libc.ID(),
+		libc:     libc,
+		interval: 1,
 	}
-	c.MemLimit = 2147483648
 	return c
 }
 
@@ -43,22 +50,19 @@ func (c *Runc) Stream() chan Metrics {
 
 func (c *Runc) run() {
 	c.running = true
-	rand.Seed(int64(time.Now().Nanosecond()))
 	defer close(c.stream)
 
 	for {
-		c.CPUUtil += rand.Intn(2) * int(c.aggression)
+		stats, err := c.libc.Stats()
-		if c.CPUUtil >= 100 {
+		if err != nil {
-			c.CPUUtil = 0
+			log.Errorf("failed to collect stats for container %s:\n%s", c.id, err)
+			break
 		}
 
-		c.NetTx += rand.Int63n(60) * c.aggression
+		c.ReadCPU(stats.CgroupStats)
-		c.NetRx += rand.Int63n(60) * c.aggression
+		c.ReadMem(stats.CgroupStats)
-		c.MemUsage += rand.Int63n(c.MemLimit/512) * c.aggression
+		c.ReadNet(stats.Interfaces)
-		if c.MemUsage > c.MemLimit {
+
-			c.MemUsage = 0
-		}
-		c.MemPercent = round((float64(c.MemUsage) / float64(c.MemLimit)) * 100)
 		c.stream <- c.Metrics
 		if c.done {
 			break
@@ -68,3 +72,46 @@ func (c *Runc) run() {
 
 	c.running = false
 }
+
+func (c *Runc) ReadCPU(stats *cgroups.Stats) {
+	u := stats.CpuStats.CpuUsage
+	ncpus := float64(len(u.PercpuUsage))
+	total := float64(u.TotalUsage)
+	system := float64(getSysCPUUsage())
+
+	cpudiff := total - c.lastCpu
+	syscpudiff := system - c.lastSysCpu
+
+	c.CPUUtil = round((cpudiff / syscpudiff * 100) * ncpus)
+	c.lastCpu = total
+	c.lastSysCpu = system
+	c.Pids = int(stats.PidsStats.Current)
+}
+
+func (c *Runc) ReadMem(stats *cgroups.Stats) {
+	c.MemUsage = int64(stats.MemoryStats.Usage.Usage)
+	c.MemLimit = int64(stats.MemoryStats.Usage.Limit)
+	c.MemPercent = round((float64(c.MemUsage) / float64(c.MemLimit)) * 100)
+}
+
+func (c *Runc) ReadNet(interfaces []*libcontainer.NetworkInterface) {
+	var rx, tx int64
+	for _, network := range interfaces {
+		rx += int64(network.RxBytes)
+		tx += int64(network.TxBytes)
+	}
+	c.NetRx, c.NetTx = rx, tx
+}
+
+func (c *Runc) ReadIO(stats *cgroups.Stats) {
+	var read, write int64
+	for _, blk := range stats.BlkioStats.IoServiceBytesRecursive {
+		if blk.Op == "Read" {
+			read = int64(blk.Value)
+		}
+		if blk.Op == "Write" {
+			write = int64(blk.Value)
+		}
+	}
+	c.IOBytesRead, c.IOBytesWrite = read, write
+}