From 8eb7689a3b725a7a3ec495309ab348e8f9f5a650 Mon Sep 17 00:00:00 2001 From: Aleksei Zakharov Date: Thu, 22 Jul 2021 21:07:17 +0300 Subject: [PATCH] Add blkiotune settings as gauges: limits and bursts (#17) * Add blkiotune settings as gauges: limits and bursts * Write error once for unsupported features * Ignore cdrom once more --- libvirt_exporter.go | 285 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 284 insertions(+), 1 deletion(-) diff --git a/libvirt_exporter.go b/libvirt_exporter.go index 2432cb2..3b7b86e 100644 --- a/libvirt_exporter.go +++ b/libvirt_exporter.go @@ -165,6 +165,105 @@ var ( []string{"domain", "target_device"}, nil) + // Block IO tune parameters + // Limits + libvirtDomainBlockTotalBytesSecDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_total_bytes"), + "Total throughput limit in bytes per second", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockWriteBytesSecDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_write_bytes"), + "Write throughput limit in bytes per second", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockReadBytesSecDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_read_bytes"), + "Read throughput limit in bytes per second", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockTotalIopsSecDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_total_requests"), + "Total requests per second limit", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockWriteIopsSecDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_write_requests"), + "Write requests per second limit", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockReadIopsSecDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_read_requests"), + "Read requests per second limit", + []string{"domain", "target_device"}, + nil) + // Burst limits + libvirtDomainBlockTotalBytesSecMaxDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_total_bytes"), + "Total throughput burst limit in bytes per second", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockWriteBytesSecMaxDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_write_bytes"), + "Write throughput burst limit in bytes per second", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockReadBytesSecMaxDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_read_bytes"), + "Read throughput burst limit in bytes per second", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockTotalIopsSecMaxDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_total_requests"), + "Total requests per second burst limit", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockWriteIopsSecMaxDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_write_requests"), + "Write requests per second burst limit", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockReadIopsSecMaxDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_read_requests"), + "Read requests per second burst limit", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockTotalBytesSecMaxLengthDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_total_bytes_length_seconds"), + "Total throughput burst time in seconds", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockWriteBytesSecMaxLengthDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_write_bytes_length_seconds"), + "Write throughput burst time in seconds", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockReadBytesSecMaxLengthDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_read_bytes_length_seconds"), + "Read throughput burst time in seconds", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockTotalIopsSecMaxLengthDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_length_total_requests_seconds"), + "Total requests per second burst time in seconds", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockWriteIopsSecMaxLengthDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_length_write_requests_seconds"), + "Write requests per second burst time in seconds", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockReadIopsSecMaxLengthDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "limit_burst_length_read_requests_seconds"), + "Read requests per second burst time in seconds", + []string{"domain", "target_device"}, + nil) + libvirtDomainBlockSizeIopsSecDesc = prometheus.NewDesc( + prometheus.BuildFQName("libvirt", "domain_block_stats", "size_iops_bytes"), + "The size of IO operations per second permitted through a block device", + []string{"domain", "target_device"}, + nil) + libvirtDomainMetaInterfacesDesc = prometheus.NewDesc( prometheus.BuildFQName("libvirt", "domain_interface", "meta"), "Interfaces metadata. Source bridge, target device, interface uuid", @@ -263,8 +362,20 @@ var ( "The amount of memory in percent, that used by domain.", []string{"domain"}, nil) + + errorsMap map[string]struct{} ) +// Write message to stdout only once for the concrete error +// "err" - an error message +// "name" - name of an error, to count it +func WriteErrorOnce(err string, name string) { + if _, ok := errorsMap[name]; !ok { + log.Printf("%s", err) + errorsMap[name] = struct{}{} + } +} + // CollectDomain extracts Prometheus metrics from a libvirt domain. func CollectDomain(ch chan<- prometheus.Metric, stat libvirt.DomainStats) error { domainName, err := stat.Domain.GetName() @@ -391,7 +502,9 @@ func CollectDomain(ch chan<- prometheus.Metric, stat libvirt.DomainStats) error for _, disk := range stat.Block { var DiskSource string var Device *libvirtSchema.Disk - if disk.Name == "hdc" { + // Ugly hack to avoid getting metrics from cdrom block device + // TODO: somehow check the disk 'device' field for 'cdrom' string + if disk.Name == "hdc" || disk.Name == "hda" { continue } /* "block..path" - string describing the source of block device , @@ -514,6 +627,175 @@ func CollectDomain(ch chan<- prometheus.Metric, stat libvirt.DomainStats) error domainName, disk.Name) } + + blockIOTuneParams, err := stat.Domain.GetBlockIoTune(disk.Name, 0) + if err != nil { + lverr, ok := err.(libvirt.Error) + if !ok { + switch lverr.Code { + case libvirt.ERR_OPERATION_INVALID: + // This should be one-shot error + log.Printf("Invalid operation GetBlockIoTune: %s", err.Error()) + case libvirt.ERR_OPERATION_UNSUPPORTED: + WriteErrorOnce("Unsupported operation GetBlockIoTune: "+err.Error(), "blkiotune_unsupported") + default: + return err + } + } + } else { + if blockIOTuneParams.TotalBytesSecSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockTotalBytesSecDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.TotalBytesSec), + domainName, + disk.Name) + } + if blockIOTuneParams.ReadBytesSecSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockReadBytesSecDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.ReadBytesSec), + domainName, + disk.Name) + } + if blockIOTuneParams.WriteBytesSecSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockWriteBytesSecDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.WriteBytesSec), + domainName, + disk.Name) + } + if blockIOTuneParams.TotalIopsSecSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockTotalIopsSecDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.TotalIopsSec), + domainName, + disk.Name) + } + if blockIOTuneParams.ReadIopsSecSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockReadIopsSecDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.ReadIopsSec), + domainName, + disk.Name) + } + if blockIOTuneParams.WriteIopsSecSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockWriteIopsSecDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.WriteIopsSec), + domainName, + disk.Name) + } + if blockIOTuneParams.TotalBytesSecMaxSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockTotalBytesSecMaxDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.TotalBytesSecMax), + domainName, + disk.Name) + } + if blockIOTuneParams.ReadBytesSecMaxSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockReadBytesSecMaxDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.ReadBytesSecMax), + domainName, + disk.Name) + } + if blockIOTuneParams.WriteBytesSecMaxSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockWriteBytesSecMaxDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.WriteBytesSecMax), + domainName, + disk.Name) + } + if blockIOTuneParams.TotalIopsSecMaxSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockTotalIopsSecMaxDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.TotalIopsSecMax), + domainName, + disk.Name) + } + if blockIOTuneParams.ReadIopsSecMaxSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockReadIopsSecMaxDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.ReadIopsSecMax), + domainName, + disk.Name) + } + if blockIOTuneParams.WriteIopsSecMaxSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockWriteIopsSecMaxDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.WriteIopsSecMax), + domainName, + disk.Name) + } + if blockIOTuneParams.TotalBytesSecMaxLengthSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockTotalBytesSecMaxLengthDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.TotalBytesSecMaxLength), + domainName, + disk.Name) + } + if blockIOTuneParams.ReadBytesSecMaxLengthSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockReadBytesSecMaxLengthDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.ReadBytesSecMaxLength), + domainName, + disk.Name) + } + if blockIOTuneParams.WriteBytesSecMaxLengthSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockWriteBytesSecMaxLengthDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.WriteBytesSecMaxLength), + domainName, + disk.Name) + } + if blockIOTuneParams.TotalIopsSecMaxLengthSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockTotalIopsSecMaxLengthDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.TotalIopsSecMaxLength), + domainName, + disk.Name) + } + if blockIOTuneParams.ReadIopsSecMaxLengthSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockReadIopsSecMaxLengthDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.ReadIopsSecMaxLength), + domainName, + disk.Name) + } + if blockIOTuneParams.WriteIopsSecMaxLengthSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockWriteIopsSecMaxLengthDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.WriteIopsSecMaxLength), + domainName, + disk.Name) + } + if blockIOTuneParams.SizeIopsSecSet { + ch <- prometheus.MustNewConstMetric( + libvirtDomainBlockSizeIopsSecDesc, + prometheus.GaugeValue, + float64(blockIOTuneParams.SizeIopsSec), + domainName, + disk.Name) + } + } } // Report network interface statistics. @@ -840,6 +1122,7 @@ func main() { libvirtURI = app.Flag("libvirt.uri", "Libvirt URI from which to extract metrics.").Default("qemu:///system").String() ) kingpin.MustParse(app.Parse(os.Args[1:])) + errorsMap = make(map[string]struct{}) exporter, err := NewLibvirtExporter(*libvirtURI) if err != nil {