Although the most common drive to show high unmanaged disk space is /ngfw, it is not necessarily this drive that will show high usage. If you see this error message check the paths bellow to make sure that this is correct before troubleshooting other possible issues.
The issue was due to syslog_ng not overwriting (rotating) older logs.
################ Technical Steps ################
root@fpr:Volume# sudo su -
root@fpr:Volume# df
Filesystem 1K-blocks Used Available Use% Mounted on
rootfs 3412056 6364 3405692 1% /
devtmpfs 3437312 9336 3427976 1% /dev
tmpfs 4053836 500 4053336 1% /run
/dev/sda1 7558312 1794576 5373132 26% /mnt/boot
/dev/sda2 945144 199392 696908 23% /opt/cisco/config
/dev/sda3 945144 68436 827864 8% /opt/cisco/platform/logs
/dev/sda5 156177912 125307820 30870092 81% /opt/cisco/csp
/dev/sda4 28705788 45116 27195840 1% /var/data/cores
cgroup_root 4053836 0 4053836 0% /dev/cgroups
none 140288 90936 49352 65% /dev/shm/snort
tmpfs 1024 0 1024 0% /var/data/cores/sysdebug/tftpd_logs
root@fpr:/# lsof | grep deleted
syslog-ng 6111 root 32w REG 8,5 110027440128 807643062 /ngfw/var/log/process_stderr.log.1 (deleted)
syslog-ng 6111 root 33w REG 8,5 4116480 807643060 /ngfw/var/log/process_stdout.log.1 (deleted)
syslog-ng 6111 6112 syslog-ng root 32w REG 8,5 110027440128 807643062 /ngfw/var/log/process_stderr.log.1 (deleted)
syslog-ng 6111 6112 syslog-ng root 33w REG 8,5 4116480 807643060 /ngfw/var/log/process_stdout.log.1 (deleted)
syslog-ng 6111 37078 syslog-ng root 32w REG 8,5 110027440128 807643062 /ngfw/var/log/process_stderr.log.1 (deleted)
syslog-ng 6111 37078 syslog-ng root 33w REG 8,5 4116480 807643060 /ngfw/var/log/process_stdout.log.1 (deleted)
# Change directory to /ngfw/etc/logrotate-5min.d and check the contents of pm.logrotate file
root@fpr:Volume# cd /ngfw/etc/logrotate-5min.d
root@fpr:logrotate-5min.d# cat pm.logrotate
/var/log/process_std*.log {
missingok
compress
copytruncate
maxsize 1G
rotate 4
sharedscripts
}
# if /ngfw is missing from the file path in pm.logrotate file contents, edit the file and add it.
root@fpr:logrotate-5min.d# vi pm.logrotate
# press i to insert text and add /ngfw to the beginning of the path. Press ESC once you are done editing. Enter :wq to save and exit.
/ngfw/var/log/process_std*.log {
missingok
compress
copytruncate
maxsize 1G
rotate 4
sharedscripts
}
# Change to direcotry /ngfw/etc/logrotate-size.d and verify that /ngfw is present within the path in the file contents.
root@fpr:logrotate-size.d# cd /ngfw/etc/logrotate-size.d
root@fpr:logrotate-size.d# cat pm.logrotate
# verify that /ngfw i included in the file path.
/ngfw/var/log/process_std*.log {
missingok
compress
nocreate
}
# Next remove the pm.logrotate files from /ngfw/etc/logrotate.d and /ngfw/etc/logrotate-size.d directories and restart the syslog-ng process.
root@fpr:logrotate-size.d# rm -f /ngfw/etc/logrotate.d/pm.logrotate
root@fpr:logrotate-size.d# rm -f /ngfw/etc/logrotate-size.d/pm.logrotate
root@fpr:logrotate-size.d# /ngfw/etc/rc.d/init.d/syslog-ng restart
# Once the syslog-ng process is started initiate a logrotate job:
root@fpr:logrotate-size.d# logrotate -v /ngfw/etc/logrotate-5min.d/pm.logrotate
# Check disk usage:
root@fpr:logrotate-size.d# df
Filesystem 1K-blocks Used Available Use% Mounted on
rootfs 3412056 6368 3405688 1% /
devtmpfs 3437312 9336 3427976 1% /dev
tmpfs 4053836 500 4053336 1% /run
/dev/sda1 7558312 1794576 5373132 26% /mnt/boot
/dev/sda2 945144 199376 696924 23% /opt/cisco/config
/dev/sda3 945144 72032 824268 9% /opt/cisco/platform/logs
/dev/sda5 156177912 17857168 138320744 12% /opt/cisco/csp
/dev/sda4 28705788 45116 27195840 1% /var/data/cores
cgroup_root 4053836 0 4053836 0% /dev/cgroups
none 140288 88064 52224 63% /dev/shm/snort
tmpfs 1024 0 1024 0% /var/data/cores/sysdebug/tftpd_logs
