clustvirt/cluster/stats.go

package cluster

import (
	"git.staur.ca/stobbsm/clustvirt/lib/log"
	"git.staur.ca/stobbsm/clustvirt/lib/storagepool"
)

// ClusterStats is used to gather stats for the entire cluster
// Combined with StatsDiff, we can get some basic cluster wide stats tracking
type ClusterStats struct {
	// CPU Statistics including number of CPUs
	CPU CPUStats
	// Memory provides information about the amount of memory, including free and
	// allocated memory
	Memory MemoryStats
	// Storage provides information about storage pools, Only get's stats for active
	// pools, and will not activate pools that are not already active.
	// Trys to sort out shared file systems from local filesystems using the Type parameter
	// of Host.StoragePoolInfo
	Storage StorageStats
	// Volume provides information on allocated volumes used in the cluster
	Volume VolumeStats
	// VM provides VM specific counters for the cluster
	VM VMStats
	// Host provides Host information for the cluster
	Host HostStats
	// Network provices available networks, and how many are shared between hosts
	Network NetworkStats
	// NetIF provides information about Libvirt allocated networks, usable by the
	// libvirt cluster
	NetIF NetIFStats

	old *ClusterStats
	c   *Cluster
}

// CPUStats provides information about the number of CPUs, Cores,
// Threads, and Speed available to the cluster.
type CPUStats struct {
	Sockets   uint32
	Cores     uint32
	Threads   uint32
	Allocated uint32
	MHz       uint64
}

// MemoryStats provies information about the amount of memory, including free and
// allocated memory. Allocated is the total allocated to Guests
type MemoryStats struct {
	Total     uint64
	Free      uint64
	Buffers   uint64
	Cached    uint64
	Allocated uint64
}

// StorageStats provides information about the available storage pools in the cluster,
// including the amount of space available, allocated, and how many pools are shared
// between hosts
// All sizes are in Bytes
type StorageStats struct {
	Total    uint64
	Used     uint64
	Free     uint64
	Active   uint32
	Inactive uint32
	Pools    uint32
}

// VolumeStats provides information about the number of volumes on the cluster.
// Counts volumes in shared storage (as detmermined by StorageStats) only once
type VolumeStats struct {
	Total    uint32
	Active   uint32
	Inactive uint32
}

// VMStats provides information about the defined Virtual Machines on the cluster
type VMStats struct {
	Count   uint32
	Started uint32
	Stopped uint32
}

// HostStats provides informatoin about the number of hosts defined, and how many
// are currently available. An unavailable host will not have it's statistics counted
type HostStats struct {
	Available uint32
	Nodes     uint32
}

// NetworkStats provides informatoin about the available Host network connections,
// including bridges and ethernet devices.
type NetworkStats struct {
	Count    uint32
	Active   uint32
	Inactive uint32
}

// NetIFStats provides information about Libvirt defined networks
type NetIFStats struct {
	Count    uint32
	Common   uint32
	Active   uint32
	Inactive uint32
}

// DeviceStats provides information about the number of allocatable devices in the
// cluster. These are PCI and USB devices.
type DeviceStats struct {
	Count uint32
}

// SecretStats provides the number of secrets defined throughout the cluster.
// Shared secrets are only counted once, and are recognized by their UUID
type SecretStats struct {
	Count  uint32
	Shared uint32
}

// ClusterStats is used to gather stats for the entire cluster
type StatDiff struct {
	CPU     CPUDiff
	Memory  MemoryDiff
	Storage StorageDiff
	Volume  VolumeDiff
	VM      VMDiff
	Host    HostDiff
	Network NetworkDiff
}

type CPUDiff struct {
	Sockets   int
	Cores     int
	Threads   int
	Allocated int
	MHz       int
}
type MemoryDiff struct {
	Total     int64
	Free      int64
	Buffers   int64
	Cached    int64
	Allocated int64
}
type StorageDiff struct {
	Total    int64
	Used     int64
	Free     int64
	Active   int64
	Inactive int64
	Pools    int
}
type VolumeDiff struct {
	Total    int
	Active   int
	Inactive int
}
type VMDiff struct {
	Count   int
	Started int
	Stopped int
}
type HostDiff struct {
	Count     int
	Available int
}
type NetworkDiff struct {
	Count    int
	Active   int
	Inactive int
}

// InitStats is given a cluster, which it then uses to load the initial statistics
// Does not close connections, but uses the host connections available to the
// cluster to add statistics together.
func InitStats(c *Cluster) *ClusterStats {
	log.Info("stats.InitStats").Msg("intializing stats")
	cs := &ClusterStats{
		c: c,
	}

	cs.Update()
	return cs
}

// Update triggers the stats collector to refresh it's statistics
func (cs *ClusterStats) Update() {
	log.Info("ClusterStats.Update").Msg("copying existing stats")
	cs.old = cs.copy()

	log.Info("ClusterStats.Update").Msg("reseting stats")
	cs.reset()
	// Start looping through each host in the cluster, adding to the total
	for _, h := range cs.c.hosts {
		log.Info("ClusterStats.Update").Str("hostname", h.HostName).Msg("filling stats")
		cs.Host.Nodes++
		cs.Host.Available++

		log.Info("ClusterStats.Update").Str("hostname", h.HostName).Msg("filling host cpu stats")
		cs.CPU.Sockets += uint32(h.HostInfo.CPU.Topology.Sockets)
		cs.CPU.Cores += uint32(h.HostInfo.CPU.Topology.Cores)
		cs.CPU.Threads += uint32(h.HostInfo.CPU.Topology.Threads * h.HostInfo.CPU.Topology.Cores)
		cs.CPU.MHz += uint64(h.HostInfo.CPU.Counter.Frequency)

		log.Info("ClusterStats.Update").Str("hostname", h.HostName).Msg("filling host memmory stats")
		cs.Memory.Total += h.NodeMemory.Total
		cs.Memory.Free += h.NodeMemory.Free
		cs.Memory.Buffers += h.NodeMemory.Buffers
		cs.Memory.Cached += h.NodeMemory.Cached

		log.Info("ClusterStats.Update").Str("hostname", h.HostName).Msg("filling host storage pool stats")
		// Storage Pool counting
		cs.Storage.Pools += uint32(len(h.StoragePoolList))
		countedSharedPools := map[string]struct{}{}
		// Loop through available storage pools
		for _, sp := range h.StoragePoolList {
			if _, ok := countedSharedPools[sp.Name]; ok {
				// Already counted this shared pool, move on
				continue
			}
			if isNetworkPool(sp.Type) {
				countedSharedPools[sp.Name] = struct{}{}
			}
			cs.Storage.Active++
			cs.Storage.Total += sp.Capacity.Value
			cs.Storage.Used += sp.Allocation.Value
			cs.Storage.Free += sp.Capacity.Value - sp.Allocation.Value
		}
		log.Info("ClusterStats.Update").Str("hostname", h.HostName).Msg("filling volume stats")
		cs.Volume.Total += uint32(len(h.VolumeList))

		log.Info("ClusterStats.Update").Str("hostname", h.HostName).Msg("filling VM stats")
		// VM Count
		cs.VM.Count += uint32(len(h.VMList))
		for _, vm := range h.VMList {
			log.Info("ClusterStats.Update").Str("hostname", h.HostName).Str("vmname", vm.Name).Msg("filling vm info")
			cs.CPU.Allocated += uint32(vm.VCPU.Value)
			cs.Memory.Allocated += uint64(vm.Memory.Value)
		}

		// Network count
		cs.Network.Count += uint32(len(h.NetworkList))
		cs.Network.Inactive++
	}
}

// Diff returns a map of all the field and how they changed
func (cs *ClusterStats) Diff() StatDiff {
	return StatDiff{
		CPU: CPUDiff{
			Sockets:   int(cs.CPU.Sockets) - int(cs.old.CPU.Sockets),
			Cores:     int(cs.CPU.Cores) - int(cs.old.CPU.Cores),
			Threads:   int(cs.CPU.Threads) - int(cs.old.CPU.Threads),
			Allocated: int(cs.CPU.Allocated) - int(cs.old.CPU.Allocated),
			MHz:       int(cs.CPU.MHz) - int(cs.old.CPU.MHz),
		},
		Memory: MemoryDiff{
			Total:     int64(cs.old.Memory.Total) - int64(cs.Memory.Total),
			Free:      int64(cs.old.Memory.Free) - int64(cs.Memory.Free),
			Buffers:   int64(cs.old.Memory.Buffers) - int64(cs.Memory.Buffers),
			Cached:    int64(cs.old.Memory.Cached) - int64(cs.Memory.Cached),
			Allocated: int64(cs.old.Memory.Allocated) - int64(cs.Memory.Allocated),
		},
		Storage: StorageDiff{
			Total:    int64(cs.old.Storage.Total) - int64(cs.Storage.Total),
			Used:     int64(cs.old.Storage.Used) - int64(cs.Storage.Used),
			Free:     int64(cs.old.Storage.Free) - int64(cs.Storage.Free),
			Active:   int64(cs.old.Storage.Active) - int64(cs.Storage.Active),
			Inactive: int64(cs.old.Storage.Inactive) - int64(cs.Storage.Inactive),
			Pools:    int(cs.old.Storage.Pools) - int(cs.Storage.Pools),
		},
		Volume: VolumeDiff{
			Total:    int(cs.old.Volume.Total) - int(cs.Volume.Total),
			Active:   int(cs.old.Volume.Active) - int(cs.Volume.Active),
			Inactive: int(cs.old.Volume.Inactive) - int(cs.Volume.Inactive),
		},

		VM: VMDiff{
			Count:   int(cs.old.VM.Count) - int(cs.VM.Count),
			Started: int(cs.old.VM.Started) - int(cs.VM.Started),
			Stopped: int(cs.old.VM.Stopped) - int(cs.VM.Stopped),
		},
		Host: HostDiff{
			Count:     int(cs.old.Host.Nodes) - int(cs.Host.Nodes),
			Available: int(cs.old.Host.Available) - int(cs.Host.Available),
		},
		Network: NetworkDiff{
			Count:    int(cs.old.Network.Count) - int(cs.Network.Count),
			Active:   int(cs.old.Network.Active) - int(cs.Network.Active),
			Inactive: int(cs.old.Network.Inactive) - int(cs.Network.Inactive),
		},
	}
}

// copy the clusterstats into a new clusterstatus object for comparison purposes
func (cs *ClusterStats) copy() *ClusterStats {
	ncs := *cs
	return &ncs
}

// reset all values to zero value
func (cs *ClusterStats) reset() {
	cs.CPU.Sockets = 0
	cs.CPU.Cores = 0
	cs.CPU.Threads = 0
	cs.CPU.Allocated = 0
	cs.CPU.MHz = 0

	cs.Memory.Total = 0
	cs.Memory.Free = 0
	cs.Memory.Buffers = 0
	cs.Memory.Cached = 0
	cs.Memory.Allocated = 0

	cs.Storage.Total = 0
	cs.Storage.Used = 0
	cs.Storage.Free = 0
	cs.Storage.Active = 0
	cs.Storage.Inactive = 0
	cs.Storage.Pools = 0

	cs.Volume.Total = 0
	cs.Volume.Active = 0
	cs.Volume.Inactive = 0

	cs.VM.Count = 0
	cs.VM.Started = 0
	cs.VM.Stopped = 0

	cs.Host.Available = 0
	cs.Host.Nodes = 0

	cs.Network.Count = 0
	cs.Network.Active = 0
	cs.Network.Inactive = 0

	cs.NetIF.Count = 0
	cs.NetIF.Common = 0
	cs.NetIF.Active = 0
	cs.NetIF.Inactive = 0
}

func isNetworkPool(pooltype string) bool {
	for _, t := range storagepool.NetTypes {
		if t == pooltype {
			return true
		}
	}
	return false
}