From f7adeacb6265eb7236d15dec831b1b7f1bd880e4 Mon Sep 17 00:00:00 2001 From: John Crispin Date: Fri, 18 Sep 2026 19:34:10 +0200 Subject: [PATCH] build: hash downloaded files only after they changed check_download_integrity hashes the downloaded file on every parse of every Makefile that uses it. With large archives this costs seconds per build, for example about 1.2 s for linux-firmware, which is on the critical path of the package build. Remember the identity (inode, size, mtime, ctime) of a file that matched its expected hash, and hash it again only after it changed. The identity comes from the stat of tools/coreutils, which a Linux host does not build. Fall back to the stat of the host, and drop the cache when neither one answers, so that a missing stat costs time and never trust. Signed-off-by: John Crispin --- include/download.mk | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/include/download.mk b/include/download.mk index b9d71f2c1aa..0ea08d372c7 100644 --- a/include/download.mk +++ b/include/download.mk @@ -63,6 +63,28 @@ endef gen_sha256sum = $(shell $(MKHASH) sha256 $(DL_DIR)/$(1)) +# $(1): file name, $(2): expected hash +# Same as gen_sha256sum, but a file that matched the expected hash is only +# hashed again after it changed. Large archives are otherwise hashed on +# every parse of every Makefile that uses them. +DL_VERIFIED_DIR:=$(TMP_DIR)/dl-verified +define gen_sha256sum_cached +$(shell \ + stat_cmd="$(STAGING_DIR_HOST)/bin/stat"; \ + [ -x "$$stat_cmd" ] || stat_cmd=stat; \ + meta="$$("$$stat_cmd" -c '%i %s %Y %Z' $(DL_DIR)/$(1) 2>/dev/null)"; \ + id="$(2) $$meta"; \ + stamp="$(DL_VERIFIED_DIR)/$(1)"; \ + if [ -n "$(2)" ] && [ -n "$$meta" ] && [ "$$(cat "$$stamp" 2>/dev/null)" = "$$id" ]; then \ + echo "$(2)"; \ + else \ + hash="$$($(MKHASH) sha256 $(DL_DIR)/$(1))"; \ + [ "$$hash" != "$(2)" ] || [ -z "$$meta" ] || \ + { mkdir -p $(DL_VERIFIED_DIR); echo "$$id" > "$$stamp"; }; \ + echo "$$hash"; \ + fi) +endef + # Used in Build/CoreTargets and HostBuild/Core as an integrity check for # downloaded files. It will add a FORCE rule if the sha256 hash does not # match, so that the download can be more thoroughly handled by download.pl. @@ -71,7 +93,7 @@ define check_download_integrity $$(if $$(and $(FILE),$$(wildcard $(DL_DIR)/$(FILE)), \ $$(filter undefined,$$(flavor DownloadChecked/$(FILE)))), \ $$(eval DownloadChecked/$(FILE):=1) \ - $$(if $$(filter-out $$(call gen_sha256sum,$(FILE)),$$(expected_hash)), \ + $$(if $$(filter-out $$(call gen_sha256sum_cached,$(FILE),$$(expected_hash)),$$(expected_hash)), \ $(DL_DIR)/$(FILE): FORCE) \ ) endef