migrate data gathering to libvirtxml parsing

- instead of getting all the data the hard way, use libvirtxml
  to parse the XML from libvirt
- this makes it more accurate, and more future proof when schema
  changes occur
- add pcidb to query devices better

cluster/lock/lock.go

@@ -0,0 +1,19 @@
+// Package lock implements a locking mechanism on shared resources to ensure
+// they don't get used at the same time. There needs to be a lock for the following:
+// - VM on Host: A VM must only exist on one host at a time
+// - Storage attached to VM: Block storage can only be attached to one VM at a time
+package lock
+
+// Locker interface used to lock and unlock Lockable resources
+type Locker interface {
+	Lock(Lockable) error
+	Unlock(Lockable) error
+}
+
+// Lockable interface must be attached to lockable resources, such as
+// Virtual Machines, block devices, and host devices that can be attached
+// to virtual machines.
+type Lockable interface {
+	Locked() bool
+	HeldBy() Locker
+}

cluster/lock/volume.go

@@ -0,0 +1 @@
+package lock

cluster/stats.go

@@ -1,97 +1,169 @@
 package cluster
 
 // ClusterStats is used to gather stats for the entire cluster
+// Combined with StatsDiff, we can get some basic cluster wide stats tracking
 type ClusterStats struct {
-	CPU struct {
+	// CPU Statistics including number of CPUs
+	CPU     CPUStats
+	// Memory provides information about the amount of memory, including free and
+	// allocated memory
+	Memory  MemoryStats
+	// Storage provides information about storage pools, Only get's stats for active
+	// pools, and will not activate pools that are not already active.
+	// Trys to sort out shared file systems from local filesystems using the Type parameter
+	// of Host.StoragePoolInfo
+	Storage StorageStats
+	// Volume provides information on allocated volumes used in the cluster
+	Volume  VolumeStats
+	// VM provides VM specific counters for the cluster
+	VM      VMStats
+	// Host provides Host information for the cluster
+	Host    HostStats
+	// Network provices available networks, and how many are shared between hosts
+	Network NetworkStats
+	// NetIF provides information about Libvirt allocated networks, usable by the 
+	// libvirt cluster
+	NetIF NetIFStats
+
+	old *ClusterStats
+	c   *Cluster
+}
+
+// CPUStats provides information about the number of CPUs, Cores,
+// Threads, and Speed available to the cluster.
+type CPUStats struct {
 	Sockets   uint32
 	Cores     uint32
 	Threads   uint32
 	Allocated uint32
+	MHz       uint64
 }
-	Memory struct {
+
+// MemoryStats provies information about the amount of memory, including free and
+// allocated memory. Allocated is the total allocated to Guests
+type MemoryStats struct {
 	Total     uint64
 	Free      uint64
 	Buffers   uint64
 	Cached    uint64
 	Allocated uint64
 }
-	Storage struct {
+
+// StorageStats provides information about the available storage pools in the cluster,
+// including the amount of space available, allocated, and how many pools are shared
+// between hosts
+type StorageStats struct {
 	Total    uint64
 	Used     uint64
 	Free     uint64
 	Active   uint32
 	Inactive uint32
 	Pools    uint32
+}
 
-		Volumes struct {
+// VolumeStats provides information about the number of volumes on the cluster.
+// Counts volumes in shared storage (as detmermined by StorageStats) only once
+type VolumeStats struct {
 	Total    uint32
 	Active   uint32
 	Inactive uint32
 }
-	}
+
-	VM struct {
+// VMStats provides information about the defined Virtual Machines on the cluster
+type VMStats struct {
 	Count   uint32
 	Started uint32
 	Stopped uint32
 }
-	Host struct {
+
+// HostStats provides informatoin about the number of hosts defined, and how many
+// are currently available. An unavailable host will not have it's statistics counted
+type HostStats struct {
 	Count     uint32
 	Available uint32
+	Nodes uint32
 }
-	Network struct {
+
+// NetworkStats provides informatoin about the available Host network connections,
+// including bridges and ethernet devices.
+type NetworkStats struct {
 	Count    uint32
 	Active   uint32
 	Inactive uint32
 }
 
-	old *ClusterStats
+// NetIFStats provides information about Libvirt defined networks
-	c   *Cluster
+type NetIFStats struct {
+	Count    uint32
+	Common   uint32
+	Active   uint32
+	Inactive uint32
+}
+
+// DeviceStats provides information about the number of allocatable devices in the
+// cluster. These are PCI and USB devices.
+type DeviceStats struct {
+	Count uint32
+}
+
+// SecretStats provides the number of secrets defined throughout the cluster.
+// Shared secrets are only counted once, and are recognized by their UUID
+type SecretStats struct {
+	Count uint32
+	Shared uint32
 }
 
 // ClusterStats is used to gather stats for the entire cluster
 type StatDiff struct {
-	CPU struct {
+	CPU     CPUDiff
+	Memory  MemoryDiff
+	Storage StorageStats
+	Volume  VolumeDiff
+	VM      VMDiff
+	Host    HostDiff
+	Network NetworkDiff
+}
+
+type CPUDiff struct {
 	Sockets   int
 	Cores     int
 	Threads   int
 	Allocated int
 }
-	Memory struct {
+type MemoryDiff struct {
-		Total     int
+	Total     int64
-		Free      int
+	Free      int64
-		Buffers   int
+	Buffers   int64
-		Cached    int
+	Cached    int64
-		Allocated int
+	Allocated int64
 }
-	Storage struct {
+type StorageDiff struct {
-		Total    int
+	Total    int64
-		Used     int
+	Used     int64
-		Free     int
+	Free     int64
-		Active   int
+	Active   int64
-		Inactive int
+	Inactive int64
 	Pools    int
-
+}
-		Volumes struct {
+type VolumeDiff struct {
 	Total    int
 	Active   int
 	Inactive int
 }
-	}
+type VMDiff struct {
-	VM struct {
 	Count   int
 	Started int
 	Stopped int
 }
-	Host struct {
+type HostDiff struct {
 	Count     int
 	Available int
 }
-	Network struct {
+type NetworkDiff struct {
 	Count    int
 	Active   int
 	Inactive int
 }
-}
 
 // InitStats is given a cluster, which it then uses to load the initial statistics
 // Does not close connections, but uses the host connections available to the
@@ -144,9 +216,9 @@ func (cs *ClusterStats) Update() {
 			cs.Storage.Used += sp.Allocation
 			cs.Storage.Free += sp.Capacity - sp.Allocation
 			// Volumes in the pool
-			cs.Storage.Volumes.Total += uint32(len(sp.Volumes))
+			cs.Volume.Total += uint32(len(sp.Volumes))
 			for range sp.Volumes {
-				cs.Storage.Volumes.Active++
+				cs.Volume.Active++
 			}
 		}
 
@@ -225,9 +297,9 @@ func (cs *ClusterStats) Diff() StatDiff {
 				Active   int
 				Inactive int
 			}{
-				Total:    int(cs.old.Storage.Volumes.Total - cs.Storage.Volumes.Total),
+				Total:    int(cs.old.Volume.Total - cs.Volume.Total),
-				Active:   int(cs.old.Storage.Volumes.Active - cs.Storage.Volumes.Active),
+				Active:   int(cs.old.Volume.Active - cs.Volume.Active),
-				Inactive: int(cs.old.Storage.Volumes.Inactive - cs.Storage.Volumes.Inactive),
+				Inactive: int(cs.old.Volume.Inactive - cs.Volume.Inactive),
 			},
 		},
 		VM: struct {

go.mod

@@ -14,5 +14,7 @@ require (
 require (
 	github.com/blend/go-sdk v1.20220411.3 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/jaypipes/pcidb v1.0.0 // indirect
+	github.com/mitchellh/go-homedir v1.0.0 // indirect
 	golang.org/x/image v0.11.0 // indirect
 )

go.sum

@@ -8,6 +8,10 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF0
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/jaypipes/pcidb v1.0.0 h1:vtZIfkiCUE42oYbJS0TAq9XSfSmcsgo9IdxSm9qzYU8=
+github.com/jaypipes/pcidb v1.0.0/go.mod h1:TnYUvqhPBzCKnH34KrIX22kAeEbDCSRJ9cqLRCuNDfk=
+github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/wcharczuk/go-chart/v2 v2.1.1 h1:2u7na789qiD5WzccZsFz4MJWOJP72G+2kUuJoSNqWnE=
 github.com/wcharczuk/go-chart/v2 v2.1.1/go.mod h1:CyCAUt2oqvfhCl6Q5ZvAZwItgpQKZOkCJGb+VGv6l14=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

lib/host/lib.go

@@ -14,7 +14,6 @@ import (
 	"git.staur.ca/stobbsm/clustvirt/lib/secret"
 	"git.staur.ca/stobbsm/clustvirt/lib/storagepool"
 	"git.staur.ca/stobbsm/clustvirt/lib/storagevol"
-	"git.staur.ca/stobbsm/clustvirt/util"
 	"libvirt.org/go/libvirt"
 	"libvirt.org/go/libvirtxml"
 )
@@ -27,20 +26,8 @@ type Host struct {
 	HostName string
 	// SystemHostName is the hostname as reported by the system itself
 	SystemHostName string
-	// FreeMemory is the available free memory
-	FreeMemory uint64
 	// LibVersion is the version of Libvirt on the host
 	LibVersion uint32
-	// HostInfo provides basic HW information about a host
-	HostInfo NodeInfo
-	// HostSEVInfo provides informatoin about AMD SEV extentions available on the host
-	HostSEVInfo SEVInfo
-	// AvailableCPUTypes are the available types of CPUs that can be used for VM creation
-	AvailableCPUTypes []string
-	// NodeMemory provides basic memory information about the Host
-	NodeMemory NodeMemoryInfo
-	// StorageCapabilities is the XML representation of the hosts storage capabilities
-	StorageCapabilities string
 	// SysInfo is the XML representation of the host system information
 	SysInfo string
 	// Alive indicates if the connection is alive
@@ -49,18 +36,24 @@ type Host struct {
 	Encrypted bool
 	// Secure indicates if the connection is secure
 	Secure bool
+	// HostInfo provides basic HW information about a host
+	HostInfo libvirtxml.CapsHost
+	// NodeMemory provides basic memory information about the Host
+	NodeMemory NodeMemoryInfo
 	// VMList is the list of virtual machines available to the host
-	VMList []VMInfo
+	VMList []libvirtxml.Domain
 	// NetIfList is the list of network interfaces on the host
-	NetIfFList []NetIfInfo
+	NetIfFList []libvirtxml.Interface
 	// NetworkList is the list of defined networks on the host
-	NetworkList []NetworkInfo
+	NetworkList []libvirtxml.Network
 	// DeviceList is the list of devices on the host
-	DeviceList []DeviceInfo
+	DeviceList []libvirtxml.NodeDevice
 	// SecretList provides a list of secrets available to the host
-	SecretList []SecretInfo
+	SecretList []libvirtxml.Secret
 	// StoragePoolList provides the list of stoarge ppols available to the host
-	StoragePoolList []StoragePoolInfo
+	StoragePoolList []libvirtxml.StoragePool
+	// VolumeList is the list of volumes available on the host
+	VolumeList []libvirtxml.StorageVolume
 
 	uri      *URI
 	conn     *libvirt.Connect
@@ -68,108 +61,6 @@ type Host struct {
 	closeErr chan error
 }
 
-// DeviceInfo holds basic information for host devices
-type DeviceInfo struct {
-	Name         string
-	Capabilities []string
-	XML          string
-}
-
-// VMInfo holds basic VM information, like the name and ID
-type VMInfo struct {
-	Name string
-	ID   uint
-	UUID []byte
-	XML  string
-  Active bool
-  VCPUs uint
-  Memory uint
-	// States are the current states active on the host
-	States []guest.VMState
-}
-
-// SecretInfo doesn't let you see the contents of secrets, but does let you see what secrets have
-// been defined in a simple format.
-type SecretInfo struct {
-	UUID string
-	XML  string
-	Type string
-}
-
-// StoragePoolInfo holds basic information on storage pools
-type StoragePoolInfo struct {
-	Name       string
-	UUID       []byte
-	Type       string
-	XML        string
-	Active     bool
-	Persistent bool
-	AutoStart  bool
-	State      string
-	Capacity   uint64
-	Allocation uint64
-	Available  uint64
-  IsNet bool
-	// HAEnabled indicates if the storage pool has High Availability
-	HAEnabled bool
-	// Volumes defined in the storage pool
-	Volumes []VolumeInfo
-}
-
-// VolumeInfo holds basic information about Volumes available in storage pools
-type VolumeInfo struct {
-	Name       string
-	Key        string
-	Path       string
-	Type       string
-	Capacity   uint64
-	Allocation uint64
-	XML        string
-}
-
-// NetIfInfo holds basic information about available network interfaces (not their connections, the devices themselves)
-type NetIfInfo struct {
-	Name    string
-	MacAddr string
-	XML     string
-}
-
-// NetworkInfo holds basic information about network connections
-type NetworkInfo struct {
-	Name string
-	UUID []byte
-	XML  string
-  Active bool
-	// NetIf is the network interface this connection is applied to
-	NetIf NetIfInfo
-}
-
-// NodeInfo represents the basic HW info for a host node
-type NodeInfo struct {
-	// livirt.NodeInfo section
-	Model   string
-	Memory  uint64
-	Cpus    uint
-	MHz     uint
-	Nodes   uint32
-	Sockets uint32
-	Cores   uint32
-	Threads uint32
-}
-
-// SEVInfo provides information about AMD SEV support
-type SEVInfo struct {
-	// livirt.NodeSEVParameters section
-	SEVEnabled      bool
-	PDH             string
-	CertChain       string
-	CBitPos         uint
-	ReducedPhysBits uint
-	MaxGuests       uint
-	MaxEsGuests     uint
-	CPU0ID          string
-}
-
 // NodeMemoryInfo provides statistis about node memory usage from libvirt.NodeMemoryStats
 type NodeMemoryInfo struct {
 	Total   uint64
@@ -234,7 +125,7 @@ func (h *Host) Close() error {
 // private methods that load the different informational parts
 
 func (h *Host) getInfo() {
-	var wg = new(sync.WaitGroup)
+	wg := new(sync.WaitGroup)
 
 	infoFuncs := []func(){
 		h.getDevicesInfo,
@@ -242,7 +133,6 @@ func (h *Host) getInfo() {
 		h.getIfaceInfo,
 		h.getNetsInfo,
 		h.getNodeInfo,
-		//		h.getSEVInfo,
 		//		h.getSecretsInfo,
 		//		h.getStoragePools,
 	}
@@ -432,34 +322,6 @@ func (h *Host) getNodeInfo() {
 	}
 }
 
-func (h *Host) getSEVInfo() {
-	// getSEVInfo
-	h.HostSEVInfo.SEVEnabled = true
-	ns, err := h.conn.GetSEVInfo(0)
-	if err != nil {
-		log.Println(err)
-		lverr, ok := err.(libvirt.Error)
-		if ok {
-			switch lverr.Code {
-			case 84:
-				log.Println("SEV functions not supported")
-				h.HostSEVInfo.SEVEnabled = false
-			default:
-				log.Println("Error encountered", lverr.Error())
-			}
-		}
-	}
-	if h.HostSEVInfo.SEVEnabled {
-		h.HostSEVInfo.PDH = util.SetNotSet(ns.PDH, ns.PDHSet)
-		h.HostSEVInfo.CertChain = util.SetNotSet(ns.CertChain, ns.CertChainSet)
-		h.HostSEVInfo.CBitPos = ns.CBitPos
-		h.HostSEVInfo.ReducedPhysBits = ns.ReducedPhysBits
-		h.HostSEVInfo.MaxGuests = ns.MaxGuests
-		h.HostSEVInfo.MaxEsGuests = ns.MaxEsGuests
-		h.HostSEVInfo.CPU0ID = util.SetNotSet(ns.CPU0ID, ns.CPU0IDSet)
-	}
-}
-
 func (h *Host) getDomainInfo() {
 	// getDomainInfo
 	doms, err := h.conn.ListAllDomains(0)
@@ -571,6 +433,12 @@ func (h *Host) getDevicesInfo() {
 			if h.DeviceList[i].XML, err = dev.GetXMLDesc(0); err != nil {
 				log.Println(err)
 			}
+			dx := &libvirtxml.NodeDevice{}
+			if err != dx.Unmarshal(h.DeviceList[i].XML); err != nil {
+				log.Println(err)
+			}
+			h.DeviceList[i].Driver = dx.Driver.Name
+			dx.Capability.PCI.Class
 
 			dev.Free()
 		}

util/pcidb.go

@@ -0,0 +1,48 @@
+package util
+
+import (
+	"log"
+
+	"github.com/jaypipes/pcidb"
+)
+
+var (
+	pcidbInitDone = false
+	db            *pcidb.PCIDB
+)
+
+const (
+	pcidbNOTFOUND string = `NOTFOUND`
+	pcidbNODB string = `NODBFOUND`
+)
+
+func initPCIDB() {
+	var err error
+
+	// Attempt to use local sources first, fallback to network if
+	// local sources aren't found
+	db, err = pcidb.New()
+	if err != nil {
+		log.Printf("warning: couldn't use local pcidb cache: %s", err)
+		log.Println("falling back to downloading database")
+		db, err = pcidb.New(pcidb.WithEnableNetworkFetch())
+		if err != nil {
+			log.Println("error: couldn't get pcidb. no more fallbacks available, will not be able to query the pcidb")
+		}
+	}
+	pcidbInitDone = true
+}
+
+func GetPCIClass(id string) string {
+	if !pcidbInitDone {
+		initPCIDB()
+	}
+	if pcidbInitDone && db == nil {
+		log.Println("unable to access pcidb")
+		return pcidbNODB
+	}
+	if class, ok := db.Classes[id]; ok {
+		return class.Name
+	}
+	return pcidbNOTFOUND
+}

view/cluster.templ

@@ -10,12 +10,17 @@ import (
 templ ClusterInfo(cs *cluster.ClusterStats, diff cluster.StatDiff, navbar []components.NavItem) {
 	@layouts.Manager("ClustVirt", "Cluster Manager", navbar) {
 		<h3>Cluster Stats</h3>
+	}
+}
+
+templ CPUStats() {
 	<table class={ "table-auto", "w-full" }>
 		<caption class={ "caption-top" }>
 			CPU stats
 		</caption>
 		<thead>
 			<tr>
+				<th></th>
 				<th>Sockets</th>
 				<th>Cores</th>
 				<th>Threads</th>
@@ -24,6 +29,9 @@ templ ClusterInfo(cs *cluster.ClusterStats, diff cluster.StatDiff, navbar []comp
 		</thead>
 		<tbody>
 			<tr>
+				<td>
+					Latest
+				</td>
 				<td>
 					{ fmt.Sprint(cs.CPU.Sockets) }
 				</td>
@@ -38,6 +46,9 @@ templ ClusterInfo(cs *cluster.ClusterStats, diff cluster.StatDiff, navbar []comp
 				</td>
 			</tr>
 			<tr>
+				<td>
+					Change
+				</td>
 				<td>
 					{ fmt.Sprint(diff.CPU.Sockets) }
 				</td>
@@ -54,4 +65,3 @@ templ ClusterInfo(cs *cluster.ClusterStats, diff cluster.StatDiff, navbar []comp
 		</tbody>
 	</table>
 }
-}