duperemove: add new package

duperemove is a simple tool for finding duplicated
extents and submitting them for deduplication. It
will de-dupe results on btrfs.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Signed-off-by: Keith T. Garner <kgarner@kgarner.com>

utils/duperemove/Makefile

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+include $(TOPDIR)/rules.mk
+
+PKG_NAME:=duperemove
+PKG_VERSION:=0.15.2
+PKG_RELEASE:=1
+
+PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
+PKG_SOURCE_URL:=https://codeload.github.com/markfasheh/duperemove/tar.gz/v$(PKG_VERSION)?
+PKG_HASH:=27809aa91b7b9b7d0810e5329614bf80af2c48e917781e682a3fbcf61fa274da
+
+PKG_MAINTAINER:=Keith Garner <kgarner@kgarner.com>
+PKG_LICENSE:=GPL-2.0-only
+PKG_LICENSE_FILES:=LICENSE
+
+PKG_BUILD_DEPENDS:=xxhash libbsd
+PKG_BUILD_PARALLEL:=1
+
+include $(INCLUDE_DIR)/package.mk
+
+# Pass VERSION/IS_RELEASE to the source's Makefile via MAKE_FLAGS, appended
+# after the include since package-defaults.mk assigns MAKE_FLAGS with "=".
+# A plain "export VERSION=" doesn't work: Package/Default reassigns the
+# top-level VERSION to the ipk version string, clobbering it.
+MAKE_FLAGS += VERSION=$(PKG_VERSION) IS_RELEASE=1
+
+define Package/duperemove
+  SECTION:=utils
+  CATEGORY:=Utilities
+  SUBMENU:=Filesystem
+  TITLE:=Duplicate file removal tool
+  URL:=https://markfasheh.github.io/duperemove/
+  DEPENDS:= \
+    +glib2 \
+    +libatomic \
+    +libblkid \
+    +libmount \
+    +libsqlite3 \
+    +libuuid \
+    +lscpu
+endef
+
+define Package/duperemove/description
+  Duperemove is a simple tool for finding duplicated extents and submitting
+  them for deduplication. When given a list of files it will hash their
+  contents on an extent by extent basis and compare those hashes to each
+  other, finding and categorizing extents that match each other. Optionally,
+  a per-block hash can be applied for further duplication lookup. When given
+  the -d option, duperemove will submit those extents for deduplication using
+  the Linux kernel FIDEDUPRANGE ioctl.
+endef
+
+define Package/duperemove/install
+	$(INSTALL_DIR) $(1)/usr/bin
+	$(INSTALL_BIN) $(PKG_BUILD_DIR)/duperemove $(1)/usr/bin
+	$(INSTALL_BIN) $(PKG_BUILD_DIR)/hashstats $(1)/usr/bin
+	$(INSTALL_BIN) $(PKG_BUILD_DIR)/btrfs-extent-same $(1)/usr/bin
+endef
+
+$(eval $(call BuildPackage,duperemove))

utils/duperemove/test-version.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# shellcheck shell=busybox
+
+case "$PKG_NAME" in
+duperemove)
+	# btrfs-extent-same has no --version/--help, so it cannot use the
+	# generic per-executable version check; override it here and
+	# verify the two binaries that do report a version.
+	duperemove --version 2>&1 | grep -F "$PKG_VERSION" || exit 1
+	hashstats --version 2>&1 | grep -F "$PKG_VERSION" || exit 1
+	;;
+*)
+	echo "Untested package: $PKG_NAME" >&2
+	exit 1
+	;;
+esac

utils/duperemove/test.sh

@@ -0,0 +1,61 @@
+#!/bin/sh
+
+# shellcheck shell=busybox
+
+case "$1" in
+duperemove)
+	# btrfs-extent-same has no --version/--help; it prints usage and
+	# exits 1 when run without the required file/offset arguments.
+	btrfs-extent-same 2>&1 | grep -F "Usage:" || exit 1
+
+	# duperemove only scans a path whose filesystem it can identify: off
+	# btrfs it rejects devices with major 0 (tmpfs, overlayfs) and
+	# otherwise needs libblkid to find a UUID for the backing device.
+	# The CI container's filesystem usually fails one of those, and the
+	# path is then skipped silently (exit 0, nothing hashed). So the
+	# scan is only checked end to end where the filesystem is
+	# supported; elsewhere we require duperemove's own refusal message,
+	# so a genuine scan regression still fails.
+	#
+	# Keep the hashfile outside the scanned directory so duperemove
+	# does not pick up its own hashfile/WAL/SHM files as scan targets.
+	# Both are absolute paths under the physical CWD (pwd -P), since
+	# duperemove canonicalizes scanned paths with realpath() before
+	# recording them and the lookup below must match that spelling. A CWD
+	# of "/" would otherwise yield "//name", so strip the trailing slash.
+	cwd="$(pwd -P)"
+	cwd="${cwd%/}"
+	dir="$(mktemp -d "$cwd/duperemove-test.XXXXXX")"
+	hashfile="$(mktemp "$cwd/duperemove-test-hashes.XXXXXX")"
+	trap 'rm -rf "$dir" "$hashfile" "$hashfile-wal" "$hashfile-shm"' EXIT
+
+	# Two identical files give duperemove real duplicate extents to
+	# hash, exercising the same code path (including the lscpu-based
+	# core count detection) that a real scan would use.
+	head -c 1048576 /dev/urandom > "$dir/a"
+	cp "$dir/a" "$dir/b"
+
+	# --debug is what makes duperemove explain why it skipped a path.
+	out="$(duperemove -r --debug --hashfile="$hashfile" "$dir" 2>&1)" || {
+		echo "$out"
+		exit 1
+	}
+	[ -s "$hashfile" ] || exit 1
+
+	if hashstats -l "$hashfile" 2>&1 | grep -F "$dir/a"; then
+		exit 0
+	fi
+
+	if echo "$out" | grep -E 'unsupported filesystem|could not get uuid|unable to find the mount infos'; then
+		echo "scan skipped: duperemove does not support this filesystem"
+		exit 0
+	fi
+
+	echo "$out"
+	exit 1
+	;;
+*)
+	echo "Untested package: $1" >&2
+	exit 1
+	;;
+esac