diff --git a/utils/duperemove/Makefile b/utils/duperemove/Makefile new file mode 100644 index 0000000000..d7004d27c0 --- /dev/null +++ b/utils/duperemove/Makefile @@ -0,0 +1,61 @@ +# SPDX-License-Identifier: GPL-2.0-only + +include $(TOPDIR)/rules.mk + +PKG_NAME:=duperemove +PKG_VERSION:=0.15.2 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/markfasheh/duperemove/tar.gz/v$(PKG_VERSION)? +PKG_HASH:=27809aa91b7b9b7d0810e5329614bf80af2c48e917781e682a3fbcf61fa274da + +PKG_MAINTAINER:=Keith Garner +PKG_LICENSE:=GPL-2.0-only +PKG_LICENSE_FILES:=LICENSE + +PKG_BUILD_DEPENDS:=xxhash libbsd +PKG_BUILD_PARALLEL:=1 + +include $(INCLUDE_DIR)/package.mk + +# Pass VERSION/IS_RELEASE to the source's Makefile via MAKE_FLAGS, appended +# after the include since package-defaults.mk assigns MAKE_FLAGS with "=". +# A plain "export VERSION=" doesn't work: Package/Default reassigns the +# top-level VERSION to the ipk version string, clobbering it. +MAKE_FLAGS += VERSION=$(PKG_VERSION) IS_RELEASE=1 + +define Package/duperemove + SECTION:=utils + CATEGORY:=Utilities + SUBMENU:=Filesystem + TITLE:=Duplicate file removal tool + URL:=https://markfasheh.github.io/duperemove/ + DEPENDS:= \ + +glib2 \ + +libatomic \ + +libblkid \ + +libmount \ + +libsqlite3 \ + +libuuid \ + +lscpu +endef + +define Package/duperemove/description + Duperemove is a simple tool for finding duplicated extents and submitting + them for deduplication. When given a list of files it will hash their + contents on an extent by extent basis and compare those hashes to each + other, finding and categorizing extents that match each other. Optionally, + a per-block hash can be applied for further duplication lookup. When given + the -d option, duperemove will submit those extents for deduplication using + the Linux kernel FIDEDUPRANGE ioctl. +endef + +define Package/duperemove/install + $(INSTALL_DIR) $(1)/usr/bin + $(INSTALL_BIN) $(PKG_BUILD_DIR)/duperemove $(1)/usr/bin + $(INSTALL_BIN) $(PKG_BUILD_DIR)/hashstats $(1)/usr/bin + $(INSTALL_BIN) $(PKG_BUILD_DIR)/btrfs-extent-same $(1)/usr/bin +endef + +$(eval $(call BuildPackage,duperemove)) diff --git a/utils/duperemove/test-version.sh b/utils/duperemove/test-version.sh new file mode 100755 index 0000000000..6b5730f031 --- /dev/null +++ b/utils/duperemove/test-version.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# shellcheck shell=busybox + +case "$PKG_NAME" in +duperemove) + # btrfs-extent-same has no --version/--help, so it cannot use the + # generic per-executable version check; override it here and + # verify the two binaries that do report a version. + duperemove --version 2>&1 | grep -F "$PKG_VERSION" || exit 1 + hashstats --version 2>&1 | grep -F "$PKG_VERSION" || exit 1 + ;; +*) + echo "Untested package: $PKG_NAME" >&2 + exit 1 + ;; +esac diff --git a/utils/duperemove/test.sh b/utils/duperemove/test.sh new file mode 100755 index 0000000000..41e68025ea --- /dev/null +++ b/utils/duperemove/test.sh @@ -0,0 +1,61 @@ +#!/bin/sh + +# shellcheck shell=busybox + +case "$1" in +duperemove) + # btrfs-extent-same has no --version/--help; it prints usage and + # exits 1 when run without the required file/offset arguments. + btrfs-extent-same 2>&1 | grep -F "Usage:" || exit 1 + + # duperemove only scans a path whose filesystem it can identify: off + # btrfs it rejects devices with major 0 (tmpfs, overlayfs) and + # otherwise needs libblkid to find a UUID for the backing device. + # The CI container's filesystem usually fails one of those, and the + # path is then skipped silently (exit 0, nothing hashed). So the + # scan is only checked end to end where the filesystem is + # supported; elsewhere we require duperemove's own refusal message, + # so a genuine scan regression still fails. + # + # Keep the hashfile outside the scanned directory so duperemove + # does not pick up its own hashfile/WAL/SHM files as scan targets. + # Both are absolute paths under the physical CWD (pwd -P), since + # duperemove canonicalizes scanned paths with realpath() before + # recording them and the lookup below must match that spelling. A CWD + # of "/" would otherwise yield "//name", so strip the trailing slash. + cwd="$(pwd -P)" + cwd="${cwd%/}" + dir="$(mktemp -d "$cwd/duperemove-test.XXXXXX")" + hashfile="$(mktemp "$cwd/duperemove-test-hashes.XXXXXX")" + trap 'rm -rf "$dir" "$hashfile" "$hashfile-wal" "$hashfile-shm"' EXIT + + # Two identical files give duperemove real duplicate extents to + # hash, exercising the same code path (including the lscpu-based + # core count detection) that a real scan would use. + head -c 1048576 /dev/urandom > "$dir/a" + cp "$dir/a" "$dir/b" + + # --debug is what makes duperemove explain why it skipped a path. + out="$(duperemove -r --debug --hashfile="$hashfile" "$dir" 2>&1)" || { + echo "$out" + exit 1 + } + [ -s "$hashfile" ] || exit 1 + + if hashstats -l "$hashfile" 2>&1 | grep -F "$dir/a"; then + exit 0 + fi + + if echo "$out" | grep -E 'unsupported filesystem|could not get uuid|unable to find the mount infos'; then + echo "scan skipped: duperemove does not support this filesystem" + exit 0 + fi + + echo "$out" + exit 1 + ;; +*) + echo "Untested package: $1" >&2 + exit 1 + ;; +esac