build: Add step "Prepare for host tools" and rewrite cache for tools

Depends on perl and libperl.so

.github/workflows/build.yml

@@ -244,9 +244,12 @@ jobs:
           rm -rf openwrt  
 
       - name: Init builder
+        env:
+          TOOLS_SRC_HASH: ${{ hashFiles('tools/**') }}
         run: |
           chmod +x *.sh
           wget https://github.com/fantastic-packages/packages/raw/refs/heads/24.10/keys/usign/53FF2B6672243D28.pub
+          echo "TOOLS_SRC_HASH=$TOOLS_SRC_HASH" >> $GITHUB_ENV
 
       - name: Update OpenWrt packages
         id: update
@@ -255,71 +258,10 @@ jobs:
           mkdir -p logs
           echo "status=success" >> $GITHUB_OUTPUT
           
-      - name: Prepare for cache
-        id: cache_prepare
-        env: 
-          TARGET: ${{ matrix.target }}
-          USE_CACHE: ${{ github.event.inputs.use_cache }}
-          CACHE_VER: ${{ github.event.inputs.cache_ver }}
-        run: |
-          if [ "$USE_CACHE" = "true" ]; then
-              rm -rf staging_dir/
-              mkdir -p staging_dir/host
-              rm -rf build_dir/
-              mkdir -p build_dir/host
-          fi
-          if [[ " kng_re r3g r3p " == *" $TARGET "* ]]; then
-              CACHE_ARCH=mips
-          else
-              CACHE_ARCH=arm
-          fi
-          echo "CACHE_ARCH = $CACHE_ARCH"
-          echo "USE_CACHE=$USE_CACHE" >> $GITHUB_ENV
-          echo "CACHE_VER=$CACHE_VER" >> $GITHUB_ENV
-          echo "CACHE_ARCH=$CACHE_ARCH" >> $GITHUB_ENV
-          echo "status=success" >> $GITHUB_OUTPUT
-
-      - name: Restore cache for host tools
-        id: tools_cache_restore
-        if: github.event.inputs.use_cache == 'true'
-        uses: actions/cache/restore@v4
-        with:
-          key: host-tools-${{ env.CACHE_VER }}-${{ env.CACHE_ARCH }}-${{ hashFiles('tools/**') }}
-          path: |
-            staging_dir/host
-            build_dir/host
-
-      - name: Fix time stamps into restored dirs
-        if: github.event.inputs.use_cache == 'true' && steps.tools_cache_restore.outputs.cache-hit == 'true'
-        run: |
-          touch staging_dir/host/.prepared
-          touch staging_dir/host/.prereq-build
-          touch staging_dir/host/stamp/.* 2>/dev/null
-          find build_dir/host -name ".prepared*"  -exec touch {} +
-          find build_dir/host -name ".configured" -exec touch {} +
-          find build_dir/host -name ".built"      -exec touch {} +
-
-      - name: Inspect restored host tools
-        if: github.event.inputs.use_cache == 'true'
-        run: |
-          echo "====== openwrt-native.txt ======"
-          cat build_dir/host/pkgconf-*/openwrt-native.txt || true
-          echo "====== staging_dir/host ====="
-          du -sh staging_dir/host || true
-          ls -la staging_dir/host || true
-          echo "====== staging_dir/host/bin ====="
-          du -sh staging_dir/host/bin || true
-          ls -1  staging_dir/host/bin | head -20 || true
-          echo "====== build_dir/host ======="
-          du -sh build_dir/host || true
-          ls -1  build_dir/host | head -20 || true
-          echo "====== path info ========"
-          grep -R "/home/runner" build_dir/host | head
-
       - name: Configure
         id: configure
         if: steps.update.outputs.status == 'success'
-        env: 
+        env:
           TARGET: ${{ matrix.target }}
           FW_VER: ${{ needs.check.outputs.fw_ver }}
         run: |
@@ -356,6 +298,121 @@ jobs:
           cp -f .config logs/config1
           cp -f .config config1
 
+      - name: Prepare for host tools
+        id: tools_prepare
+        env: 
+          TARGET: ${{ matrix.target }}
+          USE_CACHE: ${{ github.event.inputs.use_cache }}
+          CACHE_VER: ${{ github.event.inputs.cache_ver }}
+        run: |
+          echo USE_CACHE = $USE_CACHE
+          HOST_TOOLS_STG_DIR=staging_dir/host
+          HOST_TOOLS_BLD_DIR=build_dir/host
+          rm -rf staging_dir/
+          rm -rf build_dir/
+          mkdir -p staging_dir/host
+          mkdir -p build_dir/host
+          ########### check tools Makefile ############
+          if grep -q '(CONFIG_USES_MINOR),y) += yafut' tools/Makefile ; then
+              echo WARN: file tools/Makefile allowed compile yafut
+              echo ERROR: yafut compile not supported !!!
+              exit 51
+          fi
+          ########### get hash of dir with tools Makefiles ############
+          TOOLS_SRC_CRC=$( echo $TOOLS_SRC_HASH | cksum | awk '{print $1}' )
+          echo $TOOLS_SRC_CRC > logs/host_tools_sources.crc
+          echo "TOOLS_SRC_CRC=$TOOLS_SRC_CRC" >> $GITHUB_ENV
+          ########### get prereq tools (w/o compile) ############
+          ls -la $HOST_TOOLS_DIR/bin | awk '{print $9 " -> " $11}' | sort > $HOST_TOOLS_STG_DIR/.prereq-build-list
+          TOOLS_PREREQ_CRC=$( cat $HOST_TOOLS_STG_DIR/.prereq-build-list | cksum | awk '{print $1}' )
+          cp -f $HOST_TOOLS_STG_DIR/.prereq-build-list   logs/prereq-build-list.txt
+          echo "TOOLS_PREREQ_CRC=$TOOLS_PREREQ_CRC" >> $GITHUB_ENV
+          ########### patch for tools/Makefile ###############
+          cat <<'EOF' >> tools/Makefile
+          ###
+          ### patch for getinfo
+          ###
+          TOOLS_COMPILE_LIST := $(STAGING_DIR_HOST)/.tools_compile_list
+          define gen_tools_compile_list
+          	@echo "Generating $(TOOLS_COMPILE_LIST)"
+          	@mkdir -p $(STAGING_DIR_HOST)
+          	rm -f $(STAGING_DIR_HOST)/.tools_compile_*
+          	@{ \
+          		TOOLS_STATUS=""; \
+          		for tool in $(sort $(tools-y) $(tools-)); do \
+          			T_STATUS="n"; \
+          			if echo " $(tools-y) " | grep -q " $$tool " ; then T_STATUS="y"; printf "+"; else printf " "; fi; \
+          			TOOLS_STATUS="$${TOOLS_STATUS}$${T_STATUS}" ; \
+          			printf " %s\n" "$$tool"; \
+          		done; \
+          		: > $(STAGING_DIR_HOST)/.tools_compile_$$TOOLS_STATUS; \
+          	} > $(TOOLS_COMPILE_LIST)
+          endef
+          
+          $(curdir)/getinfo: .config prereq
+          	$(call gen_tools_compile_list)
+          
+          $(curdir)/prepare: $(curdir)/getinfo
+          EOF
+          ########## end of patch tools/Makefile ############
+          make tools/getinfo
+          if [ ! -f $HOST_TOOLS_STG_DIR/.tools_compile_list ]; then
+              echo ERROR: File $HOST_TOOLS_STG_DIR/.tools_compile_list not found!
+              exit 55
+          fi
+          cp -f $HOST_TOOLS_STG_DIR/.tools_compile_list  logs/tools_compile_list.txt
+          for fn in $HOST_TOOLS_STG_DIR/.tools_compile_[yn]*; do
+              [ -e $fn ] && cp -f $fn logs/$( basename "$fn" | sed 's/^\.//' )
+          done
+          TOOLS_COMPILE_CRC=$( cat $TOOLS_STG_DIR/.tools_compile_list | cksum | awk '{print $1}' )
+          echo "TOOLS_COMPILE_CRC=$TOOLS_COMPILE_CRC" >> $GITHUB_ENV
+          ###################################################
+          if [ "$USE_CACHE" = "true" ]; then
+              echo USE_CACHE = $USE_CACHE
+          fi
+          echo "USE_CACHE=$USE_CACHE" >> $GITHUB_ENV
+          echo "CACHE_VER=$CACHE_VER" >> $GITHUB_ENV
+          echo "HOST_TOOLS_STG_DIR=$HOST_TOOLS_STG_DIR" >> $GITHUB_ENV
+          echo "HOST_TOOLS_BLD_DIR=$HOST_TOOLS_BLD_DIR" >> $GITHUB_ENV
+          echo "status=success" >> $GITHUB_OUTPUT
+
+      - name: Restore cache for host tools
+        id: tools_cache_restore
+        if: github.event.inputs.use_cache == 'true'
+        uses: actions/cache/restore@v4
+        with:
+          key: host-tools-${{ env.CACHE_VER }}-${{ env.TOOLS_SRC_CRC }}-${{ env.TOOLS_PREREQ_CRC }}-${{ env.TOOLS_COMPILE_CRC }}
+          path: |
+            staging_dir/host
+            build_dir/host
+
+      - name: Fix time stamps into restored dirs
+        if: github.event.inputs.use_cache == 'true' && steps.tools_cache_restore.outputs.cache-hit == 'true'
+        run: |
+          touch $HOST_TOOLS_STG_DIR/.prepared
+          touch $HOST_TOOLS_STG_DIR/.prereq-build
+          touch $HOST_TOOLS_STG_DIR/stamp/.* 2>/dev/null
+          find  $HOST_TOOLS_BLD_DIR -name ".prepared*"  -exec touch {} +
+          find  $HOST_TOOLS_BLD_DIR -name ".configured" -exec touch {} +
+          find  $HOST_TOOLS_BLD_DIR -name ".built"      -exec touch {} +
+
+      - name: Inspect restored host tools
+        if: github.event.inputs.use_cache == 'true'
+        run: |
+          echo "====== openwrt-native.txt ======"
+          cat $HOST_TOOLS_BLD_DIR/pkgconf-*/openwrt-native.txt || true
+          echo "====== staging_dir/host ====="
+          du -sh $HOST_TOOLS_STG_DIR || true
+          ls -la $HOST_TOOLS_STG_DIR || true
+          echo "====== staging_dir/host/bin ====="
+          du -sh $HOST_TOOLS_STG_DIR/bin || true
+          ls -1  $HOST_TOOLS_STG_DIR/bin | head -20 || true
+          echo "====== build_dir/host ======="
+          du -sh $HOST_TOOLS_BLD_DIR || true
+          ls -1  $HOST_TOOLS_BLD_DIR | head -20 || true
+          echo "====== path info ========"
+          grep -R "/home/runner" $HOST_TOOLS_BLD_DIR | head
+
       - name: Monitor memory
         run: |
           echo "CPU cores = $(nproc)"
@@ -374,16 +431,6 @@ jobs:
           make -j$(nproc) download
           echo "status=success" >> $GITHUB_OUTPUT
 
-      - name: Fix time stamps into restored dirs
-        if: github.event.inputs.use_cache == 'true' && steps.tools_cache_restore.outputs.cache-hit == 'true'
-        run: |
-          touch staging_dir/host/.prepared
-          touch staging_dir/host/.prereq-build
-          touch staging_dir/host/stamp/.* 2>/dev/null
-          find build_dir/host -name ".prepared*"  -exec touch {} +
-          find build_dir/host -name ".configured" -exec touch {} +
-          find build_dir/host -name ".built"      -exec touch {} +
-
       - name: Build tools and toolchain
         id: tools
         if: ${{ steps.configure.outputs.status == 'success' && env.FAKE_BUILD != 'true' }}
@@ -396,22 +443,29 @@ jobs:
         if: steps.tools.outputs.status == 'success'
         uses: actions/cache/save@v4
         with:
-          key: host-tools-${{ env.CACHE_VER }}-${{ env.CACHE_ARCH }}-${{ hashFiles('tools/**') }}
+          key: host-tools-${{ env.CACHE_VER }}-${{ env.TOOLS_SRC_CRC }}-${{ env.TOOLS_PREREQ_CRC }}-${{ env.TOOLS_COMPILE_CRC }}
           path: |
             staging_dir/host
             build_dir/host
 
+      - name: Show host tools stamps
+        run: |
+          ls -la staging_dir/host/stamp/
+          ls -la staging_dir/host/stamp/ > logs/host_stamp_dir.txt
+
       - name: Configure 2
         id: configure2
         if: steps.tools.outputs.status == 'success'
         env: 
           TARGET: ${{ matrix.target }}
         run: |
+          rm -rf logs_backup/
+          cp -a logs logs_backup
           make clean
+          rm -rf logs
+          cp -a logs_backup logs
           ./xmake.sh $OPT_INITRAMFS -I -t $TARGET
-          mkdir -p logs
           cp -f .config logs/config2
-          [ -f config1 ] && cp -f config1 logs/config1
 
       - name: Kernel compile
         id: kernel

r3d.config

@@ -16,7 +16,6 @@ CONFIG_TARGET_ipq806x_generic_DEVICE_xiaomi_r3d=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_alpine-fan-control=y
 CONFIG_PACKAGE_luci-app-alpine-fan-control=y
 CONFIG_PACKAGE_luci-app-temp-status=y

rt-ax52.config

@@ -11,7 +11,6 @@ CONFIG_TARGET_mediatek_filogic_DEVICE_asus_rt-ax52=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 

rt-ax57m.config

@@ -11,7 +11,6 @@ CONFIG_TARGET_mediatek_filogic_DEVICE_asus_rt-ax57m=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 

rt-ax59u.config

@@ -14,7 +14,6 @@ CONFIG_TARGET_mediatek_filogic_DEVICE_asus_rt-ax59u=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 CONFIG_PACKAGE_collectd-mod-sensors=y

rt-ax89x.config

@@ -16,7 +16,6 @@ CONFIG_TARGET_qualcommax_ipq807x_DEVICE_asus_rt-ax89x=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 CONFIG_PACKAGE_collectd-mod-sensors=y

tuf-ax4200.config

@@ -14,7 +14,6 @@ CONFIG_TARGET_mediatek_filogic_DEVICE_asus_tuf-ax4200=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 CONFIG_PACKAGE_collectd-mod-sensors=y

tuf-ax4200q.config

@@ -14,7 +14,6 @@ CONFIG_TARGET_mediatek_filogic_DEVICE_asus_tuf-ax4200q=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 CONFIG_PACKAGE_collectd-mod-sensors=y

tuf-ax6000.config

@@ -14,7 +14,6 @@ CONFIG_TARGET_mediatek_filogic_DEVICE_asus_tuf-ax6000=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 CONFIG_PACKAGE_collectd-mod-sensors=y

xmake.sh

@@ -102,22 +102,38 @@ function build_target {
 	fi
 
 	if [ 1 = 1 ]; then
+		MK_IMAGE=$XDIR/include/image.mk
 		CURDATE=$( date --utc +%y%m%d )
-		############ change images prefix ############
+		if ! grep -q "(VERSION_NUMBER))-$CURDATE" $MK_IMAGE ; then
-		# IMG_PREFIX:=$(VERSION_DIST_SANITIZED)-$(IMG_PREFIX_VERNUM)$(IMG_PREFIX_VERCODE)$(IMG_PREFIX_EXTRA)$(BOARD)$(if $(SUBTARGET),-$(SUBTARGET))
+			############ change images prefix ############
-		sed -i -e 's/^IMG_PREFIX:=.*/IMG_PREFIX:=$(VERSION_DIST_SANITIZED)-$(call sanitize,$(VERSION_NUMBER))-'$CURDATE'/g' $XDIR/include/image.mk
+			# IMG_PREFIX:=$(VERSION_DIST_SANITIZED)-$(IMG_PREFIX_VERNUM)$(IMG_PREFIX_VERCODE)$(IMG_PREFIX_EXTRA)$(BOARD)$(if $(SUBTARGET),-$(SUBTARGET))
-		echo ">>> image.mk patched !!!"
+			sed -i -e 's/^IMG_PREFIX:=.*/IMG_PREFIX:=$(VERSION_DIST_SANITIZED)-$(call sanitize,$(VERSION_NUMBER))-'$CURDATE'/g' $MK_IMAGE
+			echo ">>> image.mk patched !!! (IMG_PREFIX)"
+		fi
 	fi
 	if [ 1 = 1 ]; then
 		############ remove "squashfs" suffix ############
-		#   DEVICE_IMG_NAME = $$(DEVICE_IMG_PREFIX)-$$(1)-$$(2)
+		MK_IMAGE=$XDIR/include/image.mk
-		sed -i -e 's/.*DEVICE_IMG_NAME =.*/  DEVICE_IMG_NAME = $$(DEVICE_IMG_PREFIX)-$$(2)/g' $XDIR/include/image.mk
+		if grep -q 'DEVICE_IMG_NAME = $$(DEVICE_IMG_PREFIX)-$$(1)-$$(2)' $MK_IMAGE ; then
-		if grep "squashfs-sys" $XDIR/target/linux/mediatek/image/filogic.mk >/dev/null ; then
+			sed -i -e 's/.*DEVICE_IMG_NAME =.*/  DEVICE_IMG_NAME = $$(DEVICE_IMG_PREFIX)-$$(2)/g' $MK_IMAGE
+			echo ">>> image.mk patched !!! (DEVICE_IMG_NAME)"
+		fi
+		if grep -q "squashfs-sys" $XDIR/target/linux/mediatek/image/filogic.mk ; then
 			sed -i 's/ squashfs-sys/ sys/g' $XDIR/target/linux/mediatek/image/filogic.mk
 			sed -i 's/ squashfs-sys/ sys/g' $XDIR/target/linux/mediatek/image/mt7622.mk
 			sed -i 's/ squashfs-sys/ sys/g' $XDIR/target/linux/mediatek/image/mt7623.mk
 		fi
 	fi
+	
+	if [ 1 = 1 ]; then
+		MK_HOST_TOOLS=$XDIR/tools/Makefile
+		# tools-$(if $(CONFIG_BUILD_ALL_HOST_TOOLS)$(CONFIG_USES_MINOR),y) += yafut
+		if grep -q '(CONFIG_USES_MINOR),y)' $MK_HOST_TOOLS ; then
+			# disable build yafut - not support devices with yaffs !!!
+			sed -i 's/\$(CONFIG_USES_MINOR),y)/,y)/g' $MK_HOST_TOOLS
+			echo ">>> tools/Makefile patched !!! (disable yafut)"
+		fi
+	fi
 
 	BATADV_DIR=$XDIR/feeds/luci/protocols/luci-proto-batman-adv/htdocs/luci-static/resources/protocol
 	if [ -d $BATADV_DIR ]; then
@@ -189,7 +205,16 @@ function build_target {
 		echo ">>> dropbear patched !!! (disable MODERN_ONLY)"
 	fi
 
+	HOST_TOOLS_DIR=$XDIR/staging_dir/host
+	HOST_TOOLS_STAGE=0
+	[ -d $HOST_TOOLS_DIR ] && HOST_TOOLS_STAGE=1
+
 	make defconfig
+	
+	if [ -d $HOST_TOOLS_DIR -a $HOST_TOOLS_STAGE = 0 ]; then
+		ls -la $HOST_TOOLS_DIR/bin | awk '{print $9 " -> " $11}' | sort > $HOST_TOOLS_DIR/.prereq-build-list
+		#cat $HOST_TOOLS_DIR/.prereq-build-list | cksum | awk '{print $1}' > $HOST_TOOLS_DIR/.prereq-build-list.crc
+	fi
 
 	NSS_DRV_PPPOE_ENABLE=$( get_cfg_opt_flag $CFG NSS_DRV_PPPOE_ENABLE )
 	if [ "$NSS_DRV_PPPOE_ENABLE" = y ]; then

zen-bt8.config

@@ -16,7 +16,6 @@ CONFIG_TARGET_mediatek_filogic_DEVICE_asus_zenwifi-bt8=y
 
 ### Temp and sensors
 CONFIG_PACKAGE_lm-sensors=y
-CONFIG_PACKAGE_lm-sensors-detect=y
 CONFIG_PACKAGE_luci-app-temp-status=y
 CONFIG_PACKAGE_luci-i18n-temp-status-ru=y
 CONFIG_PACKAGE_collectd-mod-sensors=y