I would use Python for such task. It might lead to more code than a pure bash solution, but:
- it's (IMO) easier to test, just use
pytest or unitest module
- it's readable for non Linux people (well except the
get_device function which is Linux specific...)
- it's easier to get started (again IMO)
- What if you want to send some emails ? To trigger new actions ? Scripts can be enriched easily with a programming language like Python.
Since Python 3.3, shutil module comes with a function named disk_usage. It can be used to get the disk usage based on a given directory.
The minor problem is that I don't known how to easily get the name of the disk, I.E, /dev/sdb, even though it's possible to get its disk usage (using any directory mounted on /dev/sdb, in my case $HOME for example). I wrote a function called get_device for this purpose.
#!/usr/bin/env python3
import argparse
from os.path import getmtime
from shutil import disk_usage, rmtree
from sys import exit
from pathlib import Path
from typing import Iterator, Tuple
def get_device(path: Path) -> str:
"""Find the mount for a given directory. This is needed only for logging purpose."""
# Read /etc/mtab to learn about mount points
mtab_entries = Path("/etc/mtab").read_text().splitlines()
# Create a dict of mount points and devices
mount_points = dict([list(reversed(line.split(" ")[:2])) for line in mtab_entries])
# Find the mount point of given path
while path.resolve(True).as_posix() not in mount_points:
path = path.parent
# Return device associated with mount point
return mount_points[path.as_posix()]
def get_directory_and_device(path: str) -> Tuple[str, Path]:
"""Exit the process if directory does not exist."""
fs_path = Path(path)
# Path must exist
if not fs_path.exists():
print(f"ERROR: No such directory: {path}")
exit(1)
# And path must be a valid directory
if not fs_path.is_dir():
print(f"Path must be a directory and not a file: {path}")
exit(1)
# Get the device
device = get_device(fs_path)
return device, fs_path
def get_disk_usage(path: Path) -> float:
# shutil.disk_usage support Path like objects so no need to cast to string
usage = disk_usage(path)
# Get disk usage in percentage
return usage.used / usage.total * 100
def remove_file_or_directory(path: Path) -> None:
"""Remove given path, which can be a directory or a file."""
# Remove files
if path.is_file():
path.unlink()
# Recursively delete directory trees
if path.is_dir():
rmtree(path)
def find_oldest_files(
path: Path, pattern: str = "*", threshold: int = 80
) -> Iterator[Path]:
"""Iterate on the files or directories present in a directory which match given pattern."""
# List the files in the directory received as argument and sort them by age
files = sorted(path.glob(pattern), key=getmtime)
# Yield file paths until usage is lower than threshold
for file in files:
usage = get_disk_usage(path)
if usage < threshold:
break
yield file
def check_and_clean(
path: str,
threshold: int = 80,
remove: bool = False,
) -> None:
"""Main function"""
device, fspath = get_directory_and_device(path)
# shutil.disk_usage support Path like objects so no need to cast to string
usage = disk_usage(path)
# Take action if needed
if usage > threshold:
print(
f"Disk usage is greather than threshold: {usage:.2f}% > {threshold}% ({device})"
)
# Iterate over files to remove
for file in find_oldest_files(fspath, "*", threshold):
print(f"Removing file {file}")
if remove:
remove_file_or_directory(file)
def main() -> None:
parser = argparse.ArgumentParser(
description="Purge old files when disk usage is above limit."
)
parser.add_argument(
"path", help="Directory path where files should be purged", type=str
)
parser.add_argument(
"--threshold",
"-t",
metavar="T",
help="Usage threshold in percentage",
type=int,
default=80,
)
parser.add_argument(
"--remove",
"--rm",
help="Files are not removed unless --removed or --rm option is specified",
action="store_true",
default=False,
)
args = parser.parse_args()
check_and_clean(
args.path,
threshold=args.threshold,
remove=args.remove,
)
if __name__ == "__main__":
main()
If you need to orchestrate many tasks using CRON, it might be worth putting together some Python code as a library, and reuse this code across many tasks.
EDIT: I finally added the CLI part in the script, I think I'll use it myself 😅
/dev/sdb(note :sdb!=sbd- watch for typos in your scripts!) ever becomes >80% full regardless of the contents of/home/ben/ftp/surveillance/? This will always delete all of your recordings, leaving you with nothing. Better to have CCTV cameras recording to a dedicated volume that is not shared with (ie) your operating system or any other users. Ideally the cameras should manage this themselves, being aware of which files are theirs and overwriting their own oldest files when the dedicated drive space becomes full.