Commit fe6d161b authored by Joel Cooklin's avatar Joel Cooklin Committed by GitHub

Merge pull request #20 from obourdon/configurable_excluded_fs_types

Configurable excluded fs types
parents c6ca4344 2330f528
......@@ -99,21 +99,26 @@ Create a task manifest file to use snap-plugin-collector-df plugin (exemplary fi
"workflow": {
"collect": {
"metrics": {
"/intel/procfs/filesystem/rootfs/space_free": {},
"/intel/procfs/filesystem/rootfs/space_reserved": {},
"/intel/procfs/filesystem/rootfs/inodes_percent_free": {},
"/intel/procfs/filesystem/rootfs/inodes_percent_used": {},
"/intel/procfs/filesystem/rootfs/device_name": {},
"/intel/procfs/filesystem/sys_fs_cgroup/inodes_used": {}
},
"/intel/procfs/filesystem/*/space_free": {},
"/intel/procfs/filesystem/*/space_reserved": {},
"/intel/procfs/filesystem/*/inodes_percent_free": {},
"/intel/procfs/filesystem/*/inodes_percent_used": {},
"/intel/procfs/filesystem/*/device_name": {},
"/intel/procfs/filesystem/*/inodes_used": {}
},
"config": {
"/intel/procfs/filesystem": {
"proc_path": "/proc",
"excluded_fs_names": "/proc/sys/fs/binfmt_misc,/var/lib/docker/aufs",
"excluded_fs_types": "proc,binfmt_misc,fuse.gvfsd-fuse,sysfs,cgroup,fusectl,pstore,debugfs,securityfs,devpts,mqueue,hugetlbfs,nsfs,rpc_pipefs,devtmpfs,none,tmpfs,aufs"
}
},
"process": null,
"publish": [
{
"plugin_name": "file",
"config": {
"file": "/tmp/published_df"
"file": "/tmp/published_df.log"
}
}
]
......
......@@ -28,6 +28,7 @@ import (
"os"
"path"
"strings"
"sync"
"syscall"
"time"
......@@ -44,11 +45,15 @@ const (
// PluginName df collector plugin name
PluginName = "df"
// Version of plugin
Version = 3
Version = 4
nsVendor = "intel"
nsClass = "procfs"
nsType = "filesystem"
ProcPath = "proc_path"
ExcludedFSNames = "excluded_fs_names"
ExcludedFSTypes = "excluded_fs_types"
)
var (
......@@ -72,7 +77,11 @@ var (
"device_name",
"device_type",
}
invalidFSTypes = []string{
dfltExcludedFSNames = []string{
"/proc/sys/fs/binfmt_misc",
"/var/lib/docker/aufs",
}
dfltExcludedFSTypes = []string{
"proc",
"binfmt_misc",
"fuse.gvfsd-fuse",
......@@ -90,13 +99,19 @@ var (
"devtmpfs",
"none",
"tmpfs",
"aufs",
}
)
// Function to check properness of configuration parameter
// and set plugin attribute accordingly
func (p *dfCollector) setProcPath(cfg interface{}) error {
procPath, err := config.GetConfigItem(cfg, "proc_path")
p.initializedMutex.Lock()
defer p.initializedMutex.Unlock()
if p.initialized {
return nil
}
procPath, err := config.GetConfigItem(cfg, ProcPath)
if err == nil && len(procPath.(string)) > 0 {
procPathStats, err := os.Stat(procPath.(string))
if err != nil {
......@@ -107,6 +122,27 @@ func (p *dfCollector) setProcPath(cfg interface{}) error {
}
p.proc_path = procPath.(string)
}
excludedFSNames, err := config.GetConfigItem(cfg, ExcludedFSNames)
if err == nil {
if len(excludedFSNames.(string)) > 0 {
p.excluded_fs_names = strings.Split(excludedFSNames.(string), ",")
} else {
p.excluded_fs_names = []string{}
}
} else {
p.excluded_fs_names = dfltExcludedFSNames
}
excludedFSTypes, err := config.GetConfigItem(cfg, ExcludedFSTypes)
if err == nil {
if len(excludedFSTypes.(string)) > 0 {
p.excluded_fs_types = strings.Split(excludedFSTypes.(string), ",")
} else {
p.excluded_fs_types = []string{}
}
} else {
p.excluded_fs_types = dfltExcludedFSTypes
}
p.initialized = true
return nil
}
......@@ -117,7 +153,7 @@ func (p *dfCollector) GetMetricTypes(cfg plugin.ConfigType) ([]plugin.MetricType
for _, kind := range metricsKind {
mts = append(mts, plugin.MetricType{
Namespace_: core.NewNamespace(namespacePrefix...).
AddDynamicElement("filesystem", "name of filesystem").
AddDynamicElement(nsType, "name of filesystem").
AddStaticElement(kind),
Description_: "dynamic filesystem metric: " + kind,
})
......@@ -132,49 +168,100 @@ func (p *dfCollector) CollectMetrics(mts []plugin.MetricType) ([]plugin.MetricTy
if err != nil {
return nil, err
}
metrics := []plugin.MetricType{}
curTime := time.Now()
dfms, err := p.stats.collect(p.proc_path)
dfms, err := p.stats.collect(p.proc_path, p.excluded_fs_names, p.excluded_fs_types)
if err != nil {
return metrics, fmt.Errorf(fmt.Sprintf("Unable to collect metrics from df: %s", err))
}
for _, m := range mts {
ns := m.Namespace()
lns := len(ns)
if lns < 5 {
return nil, fmt.Errorf("Wrong namespace length %d", lns)
if lns < 4 {
return nil, fmt.Errorf("Wrong namespace length %d: should be at least 4", lns)
}
if ns[lns-2].Value == "*" {
for _, dfm := range dfms {
kind := ns[lns-1].Value
ns1 := core.NewNamespace(createNamespace(dfm.MountPoint, kind)...)
ns1[len(ns1)-2].Name = ns[lns-2].Name
metric := plugin.MetricType{
Timestamp_: curTime,
Namespace_: ns1,
// We can request all metrics for all devices in one shot
// using namespace /intel/procfs/filesystem/*
if lns == 4 {
if ns[lns-1].Value != "*" {
return nil, fmt.Errorf("Namespace should contain wildcard")
}
for _, kind := range metricsKind {
for _, dfm := range dfms {
metric := createMetric(
core.NewNamespace(
createNamespace(dfm.MountPoint, kind)...),
curTime)
fillMetric(kind, dfm, &metric)
metrics = append(metrics, metric)
}
fillMetric(kind, dfm, &metric)
metrics = append(metrics, metric)
}
} else {
for _, dfm := range dfms {
if ns[lns-2].Value == dfm.MountPoint {
metric := plugin.MetricType{
Timestamp_: curTime,
Namespace_: ns,
} else if ns[lns-2].Value == "*" {
// namespace /intel/procfs/filesystem/*/<metric>
kind := ns[lns-1].Value
// <metric> is also wildcard => get them all
if kind == "*" {
for _, skind := range metricsKind {
for _, dfm := range dfms {
metric := createMetric(
core.NewNamespace(
createNamespace(dfm.MountPoint, skind)...),
curTime)
fillMetric(skind, dfm, &metric)
metrics = append(metrics, metric)
}
kind := ns[lns-1].Value
}
} else {
// <metric> is not wildcard => getonly matching metrics
for _, dfm := range dfms {
metric := createMetric(
core.NewNamespace(
createNamespace(dfm.MountPoint, kind)...),
curTime)
fillMetric(kind, dfm, &metric)
metrics = append(metrics, metric)
}
}
} else {
// namespace /intel/procfs/filesystem/<fs>/<metric>
kind := ns[lns-1].Value
// <metric> is also wildcard => get them all
if kind == "*" {
for _, skind := range metricsKind {
for _, dfm := range dfms {
if ns[lns-2].Value == dfm.MountPoint {
metric := createMetric(
core.NewNamespace(
createNamespace(dfm.MountPoint, skind)...),
curTime)
fillMetric(skind, dfm, &metric)
metrics = append(metrics, metric)
}
}
}
} else {
for _, dfm := range dfms {
if ns[lns-2].Value == dfm.MountPoint {
metric := createMetric(ns, curTime)
fillMetric(kind, dfm, &metric)
metrics = append(metrics, metric)
}
}
}
}
}
return metrics, nil
}
func createMetric(ns core.Namespace, curTime time.Time) plugin.MetricType {
metric := plugin.MetricType{
Timestamp_: curTime,
Namespace_: ns,
}
ns[len(ns)-2].Name = nsType
return metric
}
// Function to fill metric with proper (computed) value
func fillMetric(kind string, dfm dfMetric, metric *plugin.MetricType) {
switch kind {
......@@ -219,20 +306,28 @@ func createNamespace(elt string, name string) []string {
// It returns error in case retrieval was not successful
func (p *dfCollector) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
cp := cpolicy.New()
rule, _ := cpolicy.NewStringRule("proc_path", false, "/proc")
rule, _ := cpolicy.NewStringRule(ProcPath, false, "/proc")
node := cpolicy.NewPolicyNode()
node.Add(rule)
cp.Add([]string{nsVendor, nsClass, PluginName}, node)
rule, _ = cpolicy.NewStringRule(ExcludedFSNames, false, strings.Join(dfltExcludedFSNames, ","))
node.Add(rule)
rule, _ = cpolicy.NewStringRule(ExcludedFSTypes, false, strings.Join(dfltExcludedFSTypes, ","))
node.Add(rule)
return cp, nil
}
// NewDfCollector creates new instance of plugin and returns pointer to initialized object.
func NewDfCollector() *dfCollector {
logger := log.New()
imutex := new(sync.Mutex)
return &dfCollector{
stats: &dfStats{},
logger: logger,
proc_path: procPath,
stats: &dfStats{},
logger: logger,
initializedMutex: imutex,
proc_path: procPath,
excluded_fs_names: dfltExcludedFSNames,
excluded_fs_types: dfltExcludedFSTypes,
}
}
......@@ -244,14 +339,19 @@ func Meta() *plugin.PluginMeta {
plugin.CollectorPluginType,
[]string{plugin.SnapGOBContentType},
[]string{plugin.SnapGOBContentType},
plugin.RoutingStrategy(plugin.StickyRouting),
plugin.ConcurrencyCount(1),
)
}
type dfCollector struct {
stats collector
logger *log.Logger
proc_path string
initialized bool
initializedMutex *sync.Mutex
stats collector
logger *log.Logger
proc_path string
excluded_fs_names []string
excluded_fs_types []string
}
type dfMetric struct {
......@@ -266,14 +366,13 @@ type dfMetric struct {
}
type collector interface {
collect(string) ([]dfMetric, error)
collect(string, []string, []string) ([]dfMetric, error)
}
type dfStats struct{}
func (dfs *dfStats) collect(procPath string) ([]dfMetric, error) {
func (dfs *dfStats) collect(procPath string, excluded_fs_names []string, excluded_fs_types []string) ([]dfMetric, error) {
dfms := []dfMetric{}
cpath := path.Join(procPath, "1", "mountinfo")
fh, err := os.Open(cpath)
if err != nil {
......@@ -299,48 +398,56 @@ func (dfs *dfStats) collect(procPath string) ([]dfMetric, error) {
return nil, fmt.Errorf("Wrong format %d fields found on the right side instead of 7 min", len(rightFields))
}
// Keep only meaningfull filesystems
if !invalidFS(rightFields[0]) {
var dfm dfMetric
dfm.Filesystem = rightFields[1]
dfm.FsType = rightFields[0]
dfm.UnchangedMountPoint = leftFields[4]
if leftFields[4] == "/" {
dfm.MountPoint = "rootfs"
} else {
dfm.MountPoint = strings.Replace(leftFields[4][1:], "/", "_", -1)
// Because there are mounted FS containing dots
// (like /etc/resolv.conf in Docker containers)
// and this is incompatible with Snap metric name policies
dfm.MountPoint = strings.Replace(dfm.MountPoint, ".", "_", -1)
}
stat := syscall.Statfs_t{}
err := syscall.Statfs(leftFields[4], &stat)
if err != nil {
log.Error(fmt.Sprintf("Error getting filesystem infos for %s", leftFields[4]))
continue
}
// Blocks
dfm.Blocks = (stat.Blocks * uint64(stat.Bsize)) / 1024
dfm.Available = (stat.Bavail * uint64(stat.Bsize)) / 1024
xFree := (stat.Bfree * uint64(stat.Bsize)) / 1024
dfm.Used = dfm.Blocks - xFree
percentAvailable := ceilPercent(dfm.Used, dfm.Used+dfm.Available)
dfm.Capacity = percentAvailable / 100.0
// Inodes
dfm.Inodes = stat.Files
dfm.IFree = stat.Ffree
dfm.IUsed = dfm.Inodes - dfm.IFree
percentIUsed := ceilPercent(dfm.IUsed, dfm.Inodes)
dfm.IUse = percentIUsed / 100.0
dfms = append(dfms, dfm)
if excludedFSFromList(leftFields[4], excluded_fs_names) {
log.Debug(fmt.Sprintf("Ignoring mount point %s",
leftFields[4]))
continue
}
if excludedFSFromList(rightFields[0], excluded_fs_types) {
log.Debug(fmt.Sprintf("Ignoring mount point %s with FS type %s",
leftFields[4], rightFields[0]))
continue
}
var dfm dfMetric
dfm.Filesystem = rightFields[1]
dfm.FsType = rightFields[0]
dfm.UnchangedMountPoint = leftFields[4]
if leftFields[4] == "/" {
dfm.MountPoint = "rootfs"
} else {
dfm.MountPoint = strings.Replace(leftFields[4][1:], "/", "_", -1)
// Because there are mounted FS containing dots
// (like /etc/resolv.conf in Docker containers)
// and this is incompatible with Snap metric name policies
dfm.MountPoint = strings.Replace(dfm.MountPoint, ".", "_", -1)
}
stat := syscall.Statfs_t{}
err := syscall.Statfs(leftFields[4], &stat)
if err != nil {
log.Error(fmt.Sprintf("Error getting filesystem infos for %s", leftFields[4]))
continue
}
// Blocks
dfm.Blocks = (stat.Blocks * uint64(stat.Bsize)) / 1024
dfm.Available = (stat.Bavail * uint64(stat.Bsize)) / 1024
xFree := (stat.Bfree * uint64(stat.Bsize)) / 1024
dfm.Used = dfm.Blocks - xFree
percentAvailable := ceilPercent(dfm.Used, dfm.Used+dfm.Available)
dfm.Capacity = percentAvailable / 100.0
// Inodes
dfm.Inodes = stat.Files
dfm.IFree = stat.Ffree
dfm.IUsed = dfm.Inodes - dfm.IFree
percentIUsed := ceilPercent(dfm.IUsed, dfm.Inodes)
dfm.IUse = percentIUsed / 100.0
dfms = append(dfms, dfm)
}
return dfms, nil
}
// Return true if filesystem should not be taken into account
func invalidFS(fs string) bool {
for _, v := range invalidFSTypes {
func excludedFSFromList(fs string, excludeList []string) bool {
for _, v := range excludeList {
if fs == v {
return true
}
......@@ -372,10 +479,8 @@ func ceilPercent(v uint64, t uint64) float64 {
func makeNamespace(dfm dfMetric, kind string) []string {
ns := []string{}
ns = append(ns, namespacePrefix...)
ns = append(ns, dfm.MountPoint, kind)
return ns
}
......@@ -385,6 +490,5 @@ func validateMetric(namespace []string, dfm dfMetric) bool {
if mountPoint == dfm.MountPoint {
return true
}
return false
}
......@@ -20,6 +20,7 @@ limitations under the License.
package df
import (
"errors"
"strings"
"testing"
......@@ -29,6 +30,8 @@ import (
"github.com/intelsdi-x/snap/control/plugin"
"github.com/intelsdi-x/snap/core"
"github.com/intelsdi-x/snap/core/cdata"
"github.com/intelsdi-x/snap/core/ctypes"
)
type DfPluginSuite struct {
......@@ -38,8 +41,6 @@ type DfPluginSuite struct {
}
func (dfp *DfPluginSuite) SetupSuite() {
mc := &MockCollector{}
dfms := []dfMetric{
dfMetric{
Blocks: 100,
......@@ -68,18 +69,109 @@ func (dfp *DfPluginSuite) SetupSuite() {
IUse: 0.5,
},
}
mc.On("collect", "/proc").Return(dfms, nil)
mc := &MockCollector{}
mc.On("collect", "/proc", dfltExcludedFSNames, dfltExcludedFSTypes).Return(dfms, nil)
mc.On("collect", "/dummy", dfltExcludedFSNames, dfltExcludedFSTypes).Return(dfms, errors.New("Fake error"))
dfp.mockCollector = mc
dfp.cfg = plugin.ConfigType{}
}
func (dfp *DfPluginSuite) TestGetMetricTypes() {
Convey("Given df plugin is initialized", dfp.T(), func() {
dfPlg := dfCollector{
stats: dfp.mockCollector,
}
Convey("Given df plugin is badly initialized", dfp.T(), func() {
dfPlg := NewDfCollector()
dfPlg.stats = dfp.mockCollector
node := cdata.NewNode()
node.AddItem(ProcPath, ctypes.ConfigValueStr{Value: "/dummy"})
Convey("When list of available metrics is requested", func() {
mts := []plugin.MetricType{
plugin.MetricType{
Namespace_: core.NewNamespace("intel", "procfs", "filesystem", "rootfs", "space_free"),
Config_: node,
},
}
metrics, err := dfPlg.CollectMetrics(mts)
Convey("Then error should be reported", func() {
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "no such file or directory")
So(metrics, ShouldBeNil)
})
})
})
Convey("Given df plugin is initialized", dfp.T(), func() {
dfPlg := NewDfCollector()
dfPlg.stats = dfp.mockCollector
Convey("When values for given metrics are requested", func() {
mts, err := dfPlg.GetMetricTypes(dfp.cfg)
Convey("Then no error should be reported", func() {
So(err, ShouldBeNil)
})
Convey("Then proper metrics are returned", func() {
ns := []string{}
for _, m := range mts {
ns = append(ns, m.Namespace().String())
}
So(len(mts), ShouldEqual, 14)
So(ns, ShouldContain, "/intel/procfs/filesystem/*/space_free")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/space_reserved")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/space_used")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/space_percent_free")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/space_percent_reserved")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/space_percent_used")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/inodes_free")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/inodes_reserved")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/inodes_used")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/inodes_percent_free")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/inodes_percent_reserved")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/inodes_percent_used")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/device_name")
So(ns, ShouldContain, "/intel/procfs/filesystem/*/device_type")
})
})
})
}
func (dfp *DfPluginSuite) TestCollectMetrics() {
Convey("Given df plugin is initialized", dfp.T(), func() {
dfPlg := NewDfCollector()
dfPlg.stats = dfp.mockCollector
Convey("When list of metrics is requested with too short namespace", func() {
mts := []plugin.MetricType{
plugin.MetricType{
Namespace_: core.NewNamespace("filesystem", "rootfs", "space_free"),
},
}
metrics, err := dfPlg.CollectMetrics(mts)
Convey("Then error should be reported", func() {
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "Wrong namespace length")
So(metrics, ShouldBeNil)
})
})
Convey("When list of metrics is requested with bad namespace", func() {
mts := []plugin.MetricType{
plugin.MetricType{
Namespace_: core.NewNamespace("intel", "procfs", "filesystem", "rootfs"),
},
}
metrics, err := dfPlg.CollectMetrics(mts)
Convey("Then error should be reported", func() {
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "Namespace should contain wildcard")
So(metrics, ShouldBeNil)
})
})
Convey("When list of specific metrics is requested", func() {
mts := []plugin.MetricType{
plugin.MetricType{
Namespace_: core.NewNamespace("intel", "procfs", "filesystem", "rootfs", "space_free"),
......@@ -92,6 +184,7 @@ func (dfp *DfPluginSuite) TestGetMetricTypes() {
Convey("Then no error should be reported", func() {
So(err, ShouldBeNil)
So(metrics, ShouldNotBeNil)
})
Convey("Then proper metrics are returned", func() {
......@@ -111,43 +204,367 @@ func (dfp *DfPluginSuite) TestGetMetricTypes() {
So(val, ShouldNotBeNil)
})
})
Convey("When one single available dynamic metrics is requested", func() {
mts := []plugin.MetricType{
plugin.MetricType{
Namespace_: core.NewNamespace("intel", "procfs", "filesystem", "*", "space_free"),
},
}
metrics, err := dfPlg.CollectMetrics(mts)
Convey("Then no error should be reported", func() {
So(err, ShouldBeNil)
So(metrics, ShouldNotBeNil)
})
Convey("Then proper metrics are returned", func() {
metvals := map[string]interface{}{}
for _, m := range metrics {
stat := strings.Join(m.Namespace().Strings()[3:], "/")
metvals[stat] = m.Data()
}
So(len(metrics), ShouldEqual, 2)
val, ok := metvals["rootfs/space_free"]
So(ok, ShouldBeTrue)
So(val, ShouldNotBeNil)
val, ok = metvals["big/space_free"]
So(ok, ShouldBeTrue)
So(val, ShouldNotBeNil)
})
})
Convey("When all available dynamic metrics are requested for given mountpoint", func() {
mts := []plugin.MetricType{
plugin.MetricType{
Namespace_: core.NewNamespace("intel", "procfs", "filesystem", "rootfs", "*"),
},
}
metrics, err := dfPlg.CollectMetrics(mts)
Convey("Then no error should be reported", func() {
So(err, ShouldBeNil)
So(metrics, ShouldNotBeNil)
})
Convey("Then proper metrics are returned", func() {
metvals := map[string]interface{}{}
for _, m := range metrics {
stat := strings.Join(m.Namespace().Strings()[3:], "/")
So(stat, ShouldStartWith, "rootfs")
metvals[stat] = m.Data()
}
So(len(metrics), ShouldEqual, 14)
val, ok := metvals["rootfs/space_free"]
So(ok, ShouldBeTrue)
So(val, ShouldNotBeNil)
})
})
Convey("When all available dynamic metrics are requested for all mountpoints", func() {
mts := []plugin.MetricType{
plugin.MetricType{