build: hash downloaded files only after they changed

check_download_integrity hashes the downloaded file on every parse of
every Makefile that uses it. With large archives this costs seconds per
build, for example about 1.2 s for linux-firmware, which is on the
critical path of the package build.

Remember the identity (inode, size, mtime, ctime) of a file that
matched its expected hash, and hash it again only after it changed.

The identity comes from the stat of tools/coreutils, which a Linux host
does not build. Fall back to the stat of the host, and drop the cache
when neither one answers, so that a missing stat costs time and never
trust.

Signed-off-by: John Crispin <john@phrozen.org>

include/download.mk

@@ -63,6 +63,28 @@ endef
 
 gen_sha256sum = $(shell $(MKHASH) sha256 $(DL_DIR)/$(1))
 
+# $(1): file name, $(2): expected hash
+# Same as gen_sha256sum, but a file that matched the expected hash is only
+# hashed again after it changed. Large archives are otherwise hashed on
+# every parse of every Makefile that uses them.
+DL_VERIFIED_DIR:=$(TMP_DIR)/dl-verified
+define gen_sha256sum_cached
+$(shell \
+	stat_cmd="$(STAGING_DIR_HOST)/bin/stat"; \
+	[ -x "$$stat_cmd" ] || stat_cmd=stat; \
+	meta="$$("$$stat_cmd" -c '%i %s %Y %Z' $(DL_DIR)/$(1) 2>/dev/null)"; \
+	id="$(2) $$meta"; \
+	stamp="$(DL_VERIFIED_DIR)/$(1)"; \
+	if [ -n "$(2)" ] && [ -n "$$meta" ] && [ "$$(cat "$$stamp" 2>/dev/null)" = "$$id" ]; then \
+		echo "$(2)"; \
+	else \
+		hash="$$($(MKHASH) sha256 $(DL_DIR)/$(1))"; \
+		[ "$$hash" != "$(2)" ] || [ -z "$$meta" ] || \
+			{ mkdir -p $(DL_VERIFIED_DIR); echo "$$id" > "$$stamp"; }; \
+		echo "$$hash"; \
+	fi)
+endef
+
 # Used in Build/CoreTargets and HostBuild/Core as an integrity check for
 # downloaded files.  It will add a FORCE rule if the sha256 hash does not
 # match, so that the download can be more thoroughly handled by download.pl.
@@ -71,7 +93,7 @@ define check_download_integrity
   $$(if $$(and $(FILE),$$(wildcard $(DL_DIR)/$(FILE)), \
 	       $$(filter undefined,$$(flavor DownloadChecked/$(FILE)))), \
     $$(eval DownloadChecked/$(FILE):=1) \
-    $$(if $$(filter-out $$(call gen_sha256sum,$(FILE)),$$(expected_hash)), \
+    $$(if $$(filter-out $$(call gen_sha256sum_cached,$(FILE),$$(expected_hash)),$$(expected_hash)), \
       $(DL_DIR)/$(FILE): FORCE) \
   )
 endef