#50509 Issue 50488 - Create a monitor for disk space usage
Closed 2 years ago by spichugi. Opened 2 years ago by spichugi.
spichugi/389-ds-base disk-space-mon  into  master

@@ -0,0 +1,45 @@ 

+ # --- BEGIN COPYRIGHT BLOCK ---

+ # Copyright (C) 2019 Red Hat, Inc.

+ # All rights reserved.

+ #

+ # License: GPL (version 3 or any later version).

+ # See LICENSE for details.

+ # --- END COPYRIGHT BLOCK ---

+ from lib389.monitor import MonitorDiskSpace

+ from lib389.topologies import topology_st as topo

+ 

+ 

+ def test_basic(topo):

+     """Test that the cn=disk space,cn=monitor gives at least one value

+ 

+     :id: f1962762-2c6c-4e50-97af-a00012a7486d

+     :setup: Standalone

+     :steps:

+         1. Get cn=disk space,cn=monitor entry

+         2. Check it has at least one dsDisk attribute

+         3. Check dsDisk attribute has the partition and sizes

+         4. Check the numbers are valid integers

+     :expectedresults:

+         1. It should succeed

+         2. It should succeed

+         3. It should succeed

+         4. It should succeed

+     """

+ 

+     inst = topo.standalone

+ 

+     # Turn off disk monitoring

+     disk_space_mon = MonitorDiskSpace(inst)

+     disk_str = disk_space_mon.get_disks()[0]

+ 

+     inst.log.info('Check that "partition", "size", "used", "available", "use%" words are present in the string')

+     words = ["partition", "size", "used", "available", "use%"]

+     assert all(map(lambda word: word in disk_str, words))

+ 

+     inst.log.info("Check that the sizes are numbers")

+     for word in words[1:]:

+         number = disk_str.split(f'{word}="')[1].split('"')[0]

+         try:

+             int(number)

+         except ValueError:

+             raise ValueError(f'A "{word}" value is not a number')

file modified
+51 -89
@@ -42,14 +42,13 @@ 

  #if defined(LINUX) || defined(__FreeBSD__)

  #ifdef LINUX

  #undef CTIME

- #include <sys/statfs.h>

  #endif /* linux*/

  #include <sys/param.h>

  #include <sys/mount.h>

  #else /* Linux or fbsd */

- #include <sys/statvfs.h>

  #include <sys/mnttab.h>

  #endif

+ #include <sys/statvfs.h>

  #include "slap.h"

  #include "slapi-plugin.h"

  #include "snmp_collator.h"
@@ -210,67 +209,8 @@ 

  /*

   *  Return a copy of the mount point for the specified directory

   */

- #ifdef SOLARIS

- char *

- disk_mon_get_mount_point(char *dir)

- {

-     struct mnttab mnt;

-     struct stat s;

-     dev_t dev_id;

-     FILE *fp;

- 

-     fp = fopen("/etc/mnttab", "r");

- 

-     if (fp == NULL || stat(dir, &s) != 0) {

-         return NULL;

-     }

- 

-     dev_id = s.st_dev;

- 

-     while ((0 == getmntent(fp, &mnt))) {

-         if (stat(mnt.mnt_mountp, &s) != 0) {

-             continue;

-         }

-         if (s.st_dev == dev_id) {

-             return (slapi_ch_strdup(mnt.mnt_mountp));

-         }

-     }

- 

-     return NULL;

- }

- #elif HPUX

- char *

- disk_mon_get_mount_point(char *dir)

- {

-     struct mntent *mnt;

-     struct stat s;

-     dev_t dev_id;

-     FILE *fp;

- 

-     if ((fp = setmntent("/etc/mnttab", "r")) == NULL) {

-         return NULL;

-     }

- 

-     if (stat(dir, &s) != 0) {

-         return NULL;

-     }

- 

-     dev_id = s.st_dev;

- 

-     while ((mnt = getmntent(fp))) {

-         if (stat(mnt->mnt_dir, &s) != 0) {

-             continue;

-         }

-         if (s.st_dev == dev_id) {

-             endmntent(fp);

-             return (slapi_ch_strdup(mnt->mnt_dir));

-         }

-     }

-     endmntent(fp);

  

-     return NULL;

- }

- #elif LINUX /* Linux */

+ #if LINUX

  char *

  disk_mon_get_mount_point(char *dir)

  {
@@ -304,8 +244,8 @@ 

  char *

  disk_mon_get_mount_point(char *dir)

  {

-     struct statfs sb;

-     if (statfs(dir, &sb) != 0) {

+     struct statvfs sb;

+     if (statvfs(dir, &sb) != 0) {

          return NULL;

      }

  
@@ -337,14 +277,14 @@ 

   *  We gather all the log, txn log, config, and db directories

   */

  void

- disk_mon_get_dirs(char ***list, int logs_critical __attribute__((unused)))

+ disk_mon_get_dirs(char ***list)

  {

      slapdFrontendConfig_t *config = getFrontendConfig();

      Slapi_Backend *be = NULL;

      char *cookie = NULL;

      char *dir = NULL;

  

- /* Add /var just to be safe */

+     /* Add /var just to be safe */

  #ifdef LOCALSTATEDIR

      disk_mon_add_dir(list, LOCALSTATEDIR);

  #else
@@ -362,10 +302,12 @@ 

  

      be = slapi_get_first_backend(&cookie);

      while (be) {

-         if (slapi_back_get_info(be, BACK_INFO_DIRECTORY, (void **)&dir) == LDAP_SUCCESS) { /* db directory */

+         if (slapi_back_get_info(be, BACK_INFO_DIRECTORY, (void **)&dir) == LDAP_SUCCESS) {

+             /* db directory */

              disk_mon_add_dir(list, dir);

          }

-         if (slapi_back_get_info(be, BACK_INFO_LOG_DIRECTORY, (void **)&dir) == LDAP_SUCCESS) { /*  txn log dir */

+         if (slapi_back_get_info(be, BACK_INFO_LOG_DIRECTORY, (void **)&dir) == LDAP_SUCCESS) {

+             /* txn log dir */

              disk_mon_add_dir(list, dir);

          }

          be = (backend *)slapi_get_next_backend(cookie);
@@ -374,32 +316,52 @@ 

  }

  

  /*

+  *  This function gets the stats of the directory and returns total space,

+  *  available space, and used space of the directory.

+  */

+ int32_t

+ disk_get_info(char *dir, uint64_t *total_space, uint64_t *avail_space, uint64_t *used_space)

+ {

+     int32_t rc = LDAP_SUCCESS;

+     struct statvfs buf;

+     uint64_t freeBytes = 0;

+     uint64_t blockSize = 0;

+     uint64_t blocks = 0;

+ 

+     if (statvfs(dir, &buf) != -1) {

+         LL_UI2L(freeBytes, buf.f_bavail);

+         LL_UI2L(blockSize, buf.f_bsize);

+         LL_UI2L(blocks, buf.f_blocks);

+         LL_MUL(*total_space, blocks, blockSize);

+         LL_MUL(*avail_space, freeBytes, blockSize);

+         *used_space = *total_space - *avail_space;

+     } else {

+         *total_space = 0;

+         *avail_space = 0;

+         *used_space = 0;

+         rc = -1;

+     }

+     return rc;

+ }

+ 

+ /*

   *  This function checks the list of directories to see if any are below the

-  *  threshold.  We return the the directory/free disk space of the most critical

+  *  threshold.  We return the directory/free disk space of the most critical

   *  directory.

   */

  char *

- disk_mon_check_diskspace(char **dirs, PRUint64 threshold, PRUint64 *disk_space)

+ disk_mon_check_diskspace(char **dirs, uint64_t threshold, uint64_t *disk_space)

  {

- #if defined(LINUX) || defined(__FreeBSD__)

-     struct statfs buf;

- #else

      struct statvfs buf;

- #endif

-     PRUint64 worst_disk_space = threshold;

-     PRUint64 freeBytes = 0;

-     PRUint64 blockSize = 0;

+     uint64_t worst_disk_space = threshold;

+     uint64_t freeBytes = 0;

+     uint64_t blockSize = 0;

      char *worst_dir = NULL;

      int hit_threshold = 0;

      int i = 0;

  

      for (i = 0; dirs && dirs[i]; i++) {

- #if defined(LINUX) || defined(__FreeBSD__)

-         if (statfs(dirs[i], &buf) != -1)

- #else

-         if (statvfs(dirs[i], &buf) != -1)

- #endif

-         {

+         if (statvfs(dirs[i], &buf) != -1) {

              LL_UI2L(freeBytes, buf.f_bavail);

              LL_UI2L(blockSize, buf.f_bsize);

              LL_MUL(freeBytes, freeBytes, blockSize);
@@ -444,10 +406,10 @@ 

  {

      char **dirs = NULL;

      char *dirstr = NULL;

-     PRUint64 previous_mark = 0;

-     PRUint64 disk_space = 0;

-     PRInt64 threshold = 0;

-     PRUint64 halfway = 0;

+     uint64_t previous_mark = 0;

+     uint64_t disk_space = 0;

+     int64_t threshold = 0;

+     uint64_t halfway = 0;

      time_t start = 0;

      time_t now = 0;

      int deleted_rotated_logs = 0;
@@ -500,7 +462,7 @@ 

           */

          slapi_ch_array_free(dirs);

          dirs = NULL;

-         disk_mon_get_dirs(&dirs, logging_critical);

+         disk_mon_get_dirs(&dirs);

          dirstr = disk_mon_check_diskspace(dirs, threshold, &disk_space);

          if (dirstr == NULL) {

              /*
@@ -1630,7 +1592,7 @@ 

  /**

   * Schedule more I/O for this connection, or make sure that it

   * is closed in the event loop.

-  * 

+  *

   * caller must hold c_mutex

   */

  void

@@ -88,6 +88,11 @@ 

          "cn:monitor\n"

          "aci: (target =\"ldap:///cn=monitor*\")(targetattr != \"aci || connection\")(version 3.0; acl \"monitor\"; allow( read, search, compare ) userdn = \"ldap:///anyone\";)\n",

  

+         "dn:cn=disk space,cn=monitor\n"

+         "objectclass:top\n"

+         "objectclass:extensibleObject\n"

+         "cn:disk space\n",

+ 

          "dn:cn=snmp,cn=monitor\n"

          "objectclass:top\n"

          "objectclass:extensibleObject\n"
@@ -2805,10 +2810,12 @@ 

          Slapi_DN encryption;

          Slapi_DN saslmapping;

          Slapi_DN plugins;

+         Slapi_DN diskspace;

  

          slapi_sdn_init_ndn_byref(&monitor, "cn=monitor");

          slapi_sdn_init_ndn_byref(&counters, "cn=counters,cn=monitor");

          slapi_sdn_init_ndn_byref(&snmp, "cn=snmp,cn=monitor");

+         slapi_sdn_init_ndn_byref(&diskspace, "cn=disk space,cn=monitor");

          slapi_sdn_init_ndn_byref(&root, "");

  

          slapi_sdn_init_ndn_byref(&encryption, "cn=encryption,cn=config");
@@ -2818,6 +2825,7 @@ 

          /* Search */

          dse_register_callback(pfedse, SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, &config, LDAP_SCOPE_BASE, "(objectclass=*)", read_config_dse, NULL, NULL);

          dse_register_callback(pfedse, SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, &monitor, LDAP_SCOPE_BASE, "(objectclass=*)", monitor_info, NULL, NULL);

+         dse_register_callback(pfedse, SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, &diskspace, LDAP_SCOPE_BASE, "(objectclass=*)", monitor_disk_info, NULL, NULL);

          dse_register_callback(pfedse, SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, &root, LDAP_SCOPE_BASE, "(objectclass=*)", read_root_dse, NULL, NULL);

          dse_register_callback(pfedse, SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, &monitor, LDAP_SCOPE_SUBTREE, EGG_FILTER, search_easter_egg, NULL, NULL); /* Egg */

          dse_register_callback(pfedse, SLAPI_OPERATION_SEARCH, DSE_FLAG_PREOP, &counters, LDAP_SCOPE_BASE, "(objectclass=*)", search_counters, NULL, NULL);

file modified
+48 -18
@@ -1,6 +1,6 @@ 

  /** BEGIN COPYRIGHT BLOCK

   * Copyright (C) 2001 Sun Microsystems, Inc. Used by permission.

-  * Copyright (C) 2005 Red Hat, Inc.

+  * Copyright (C) 2019 Red Hat, Inc.

   * All rights reserved.

   *

   * License: GPL (version 3 or any later version).
@@ -31,10 +31,7 @@ 

  #include "slap.h"

  #include "fe.h"

  

- #if defined(SLAPD_MONITOR_DN)

- 

- 

- int

+ int32_t

  monitor_info(Slapi_PBlock *pb __attribute__((unused)),

               Slapi_Entry *e,

               Slapi_Entry *entryAfter __attribute__((unused)),
@@ -97,14 +94,10 @@ 

      val.bv_val = buf;

      attrlist_replace(&e->e_attrs, "nbackends", vals);

  

- #ifdef THREAD_SUNOS5_LWP

-     val.bv_len = snprintf(buf, sizeof(buf), "%d", thr_getconcurrency());

-     val.bv_val = buf;

-     attrlist_replace(&e->e_attrs, "concurrency", vals);

- #endif

- 

-     /*Loop through the backends, and stuff the monitordns

-      into the entry we're sending back*/

+     /*

+      * Loop through the backends, and stuff the monitor dn's

+      * into the entry we're sending back

+      */

      attrlist_delete(&e->e_attrs, "backendmonitordn");

      cookie = NULL;

      be = slapi_get_first_backend(&cookie);
@@ -127,8 +120,47 @@ 

      return SLAPI_DSE_CALLBACK_OK;

  }

  

- #endif /* SLAPD_MONITOR_DN */

  

+ int32_t

+ monitor_disk_info (Slapi_PBlock *pb __attribute__((unused)),

+                    Slapi_Entry *e,

+                    Slapi_Entry *entryAfter __attribute__((unused)),

+                    int *returncode,

+                    char *returntext __attribute__((unused)),

+                    void *arg __attribute__((unused)))

+ {

+     int32_t rc = LDAP_SUCCESS;

+     char **dirs = NULL;

+     char buf[BUFSIZ];

+     struct berval val;

+     struct berval *vals[2];

+     uint64_t total_space;

+     uint64_t avail_space;

+     uint64_t used_space;

+ 

+     vals[0] = &val;

+     vals[1] = NULL;

+ 

+     disk_mon_get_dirs(&dirs);

+ 

+     for (uint16_t i = 0; dirs && dirs[i]; i++) {

+         rc = disk_get_info(dirs[i], &total_space, &avail_space, &used_space);

+         if (rc) {

+             slapi_log_err(SLAPI_LOG_WARNING, "monitor_disk_info",

+                           "Unable to get 'cn=disk space,cn=monitor' stats for %s\n", dirs[i]);

+         } else {

+             val.bv_len = snprintf(buf, sizeof(buf),

+                                   "partition=\"%s\" size=\"%" PRIu64 "\" used=\"%" PRIu64 "\" available=\"%" PRIu64 "\" use%%=\"%" PRIu64 "\"",

+                                   dirs[i], total_space, used_space, avail_space, used_space * 100 / total_space);

+             val.bv_val = buf;

+             attrlist_merge(&e->e_attrs, "dsDisk", vals);

+         }

+     }

+     slapi_ch_array_free(dirs);

+ 

+     *returncode = rc;

+     return SLAPI_DSE_CALLBACK_OK;

+ }

  

  /*

   * Return a malloc'd version value.
@@ -142,11 +174,9 @@ 

  

      versionstring = config_get_versionstring();

      buildnum = config_get_buildnum();

- 

      vs = slapi_ch_smprintf("%s B%s", versionstring, buildnum);

- 

-     slapi_ch_free((void **)&buildnum);

-     slapi_ch_free((void **)&versionstring);

+     slapi_ch_free_string(&buildnum);

+     slapi_ch_free_string(&versionstring);

  

      return vs;

  }

@@ -873,7 +873,8 @@ 

  /*

   * monitor.c

   */

- int monitor_info(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);

+ int32_t monitor_info(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);

+ int32_t monitor_disk_info(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry *entryAfter, int *returncode, char *returntext, void *arg);

  char *slapd_get_version_value(void);

  

  
@@ -1485,6 +1486,8 @@ 

  void slapd_do_nothing(int);

  #endif

  void slapd_wait4child(int);

+ void disk_mon_get_dirs(char ***list);

+ int32_t disk_get_info(char *dir, uint64_t *total_space, uint64_t *avail_space, uint64_t *used_space);

  

  void ns_handle_pr_read_ready(struct ns_job_t *job);

  void ns_connection_post_io_or_closing(Connection *conn);

@@ -301,7 +301,6 @@ 

  #define SLAPD_DEFAULT_LDAPI_SEARCH_BASE "dc=example,dc=com"

  #define SLAPD_DEFAULT_LDAPI_AUTO_DN     "cn=peercred,cn=external,cn=auth"

  

- #define SLAPD_MONITOR_DN "cn=monitor"

  #define SLAPD_SCHEMA_DN  "cn=schema"

  #define SLAPD_CONFIG_DN  "cn=config"

  

@@ -246,3 +246,16 @@ 

  

      def get_status(self, use_json=False):

          return self.get_attrs_vals_utf8(self._snmp_keys)

+ 

+ 

+ class MonitorDiskSpace(DSLdapObject):

+     """A class for representing "cn=disk space,cn=monitor" entry"""

+ 

+     def __init__(self, instance, dn=None):

+         super(MonitorDiskSpace, self).__init__(instance=instance, dn=dn)

+         self._dn = "cn=disk space,cn=monitor"

+ 

+     def get_disks(self):

+         """Get an information about partitions which contains a Directory Server data"""

+ 

+         return self.get_attr_vals_utf8_l("dsDisk")

Description: Create a new monitor object: cn=disk space,cn=monitor.
It contains 'dsDisk' multi-valued attribute which has a format:

dsdisk: partition="/" size="42006183936" used="35768864768" available="6237319
 168" use%="85"
dsdisk: partition="/tmp" size="1023303680" used="950198272" available="7310540
 8" use%="92"

Add MonitorDiskSpace(DSLdapObject) to monitor.py.
Add a test to check the basic functionality.
Remove unused code and its statfs.h dependency.

https://pagure.io/389-ds-base/issue/50488

Authors: spichugi, mreynolds

Reviewed by: ?

rebased onto a290a83187a2b807df9d39941ff7bf0e3068fb53

2 years ago

Compiler warning:

```
../389-ds-base/ldap/servers/slapd/daemon.c: In function ‘disk_get_info’:
../389-ds-base/ldap/servers/slapd/daemon.c:342:1: warning: control reaches end of non-void function [-Wreturn-type]
342 | }
| ^
````

It's because disk_get_info() is expected to return a int32_t, but nothing is returned.

The rest looks good to me

The patch looks good. You have my ack
If disk_get_info can not get meaningful values for a specific partition you may log a warning message.
This also prevents division if total_space==0

1 new commit added

  • Fix the return code for disk_get_info function
2 years ago

Nice catch! Thank you!

Ready for review.
If nothing will come up, I'll merge it soon because I have the acks. :)

After removing this the define from slap.h is not used anywhere, so should be removed too I guess.

After removing this the define from slap.h is not used anywhere, so should be removed too I guess.

Pagure is pretty bad with links and comments... Which piece of code exactly do you mean?
it's written - on line 13 of ldap/servers/slapd/monitor.c - and it is an empty line (there is some stuff before it but I am not sure if you meant it)

Sorry, I've apparently commented on outdated code and Pagure didn't care, sigh. I mean the SLAPD_MONITOR_DN define in slap.h, it is not used now any more.

rebased onto c778255

2 years ago

Removed it for now.
But maybe in the future, we can replace "cn=monitor" strings in the code with this definition.

Pull-Request has been merged by spichugi

2 years ago

389-ds-base is moving from Pagure to Github. This means that new issues and pull requests
will be accepted only in 389-ds-base's github repository.

This pull request has been cloned to Github as issue and is available here:
- https://github.com/389ds/389-ds-base/issues/3565

If you want to continue to work on the PR, please navigate to the github issue,
download the patch from the attachments and file a new pull request.

Thank you for understanding. We apologize for all inconvenience.

Pull-Request has been closed by spichugi

2 years ago