commit 0176f0668f78f2508bd4fe0bb2a427a40f0c1389 Author: Mads Meisner-Jensen Date: Sat May 15 14:48:49 2021 +0200 Initial commit with rpi-4, archer-c7-v2 and tl-wdr-3600-v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a59aec7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# general ignores +*~ +*.html + +# IDEs +.idea/ + +# python +__pycache__ +*.pyc + + +# project specific ignores +openwrt-imagebuilder-* +openwrt-*.img +openwrt-*.img.gz +config-backup* diff --git a/README.md b/README.md new file mode 100644 index 0000000..197fe63 --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +% OpenWRT Image Buildomatic + +# Overview + +This shell script uses the official +[OpenWrt Image Builder](https://openwrt.org/docs/guide-user/additional-software/imagebuilder) +to quickly generate/build an OpenWRT device image with extra pre-installed +packages, files or configuration included in the final image. + +This means that to build your very own, let's say Raspberry Pi 4, +OpenWRT image, you just need these three lines: + +``` + git clone https://github.com/mmeisner/openwrt-image-buildomatic + cd openwrt-image-buildomatic + ./oi-build -b -c rpi-4 +``` +...and then after a minute you will have an image you can write to an SD-card, +insert it into your RPi4 and run your Internet connection with 940Mbps +throughput, with Wireguard. + +As the saying goes: The cool guys build from source, the impatient guys +use the image builder. + +See the device configuration files in the `configs` folder for how simple +it is. + +## The OpenWRT Image Builder + +The *OpenWRT Image Builder* is like the normal OpenWRT build system except +it does not build from source code but downloads a pre-built binary image +and packages and combines those into a new image that can be used to update +your OpenWRT device; Instead of taking hours to build an image from source, +it takes mere seconds to build an image - depending on your download speed. + +The OpenWRT image builder is already very easy to use, so the only thing that +this script adds, is automated download and execution of the image builder +— plus a convenient way to specify your configuration in a simple +textual file (actually merely a shell script with a few variables). +In the configuration file you can specify: + + - **`RELEASE`** is the release to use, e.g. `21.02-rc1`, `19.07`, `snapshot`, etc. + - **`TARGET`** is the CPU architecture such as *`x86`*, + *`ath79`* (for Archer C7), *`bcm27xx`* (for Raspberry PI), etc. + - **`PROFILE`** is the device name to make an image for e.g. *`rpi4`* or *`archer-c7-v2`* + - **`PACKAGES`** is a list of additional packages to include in the image. + Full list per release is here: [OpenWrt Packages](https://openwrt.org/packages/start) + - **`FILES`** is the path to a directory containing additional files you + want to add to the final image. The directory should be organized as + a normal rootfs. It could contain some useful home-made scripts or + `uci-defaults` files for predefined configuration at first boot (see + [UCI defaults](https://openwrt.org/docs/guide-developer/uci-defaults)) + +Please note that as stated on the +[OpenWrt Image Builder](https://openwrt.org/docs/guide-user/additional-software/imagebuilder) +page, *prebuilt snapshot images do not come with any web interface or GUI*. +This means that `PACKAGES` should at least contain `luci` if you want the web GUI! + +Here are some of the OpenWRT [Release Builds](https://openwrt.org/releases/start) +and devices that the script works with (as of May 2021): + + - Releases: + - *snapshot* (snapshot, next upcoming release) + - *21.02.0-rc1* (release candidate) + - *19.07* + - Some popular devices: + - *Raspberry Pi 4* + - *Archer C7* + +OpenWRT [Index of /releases/](https://downloads.openwrt.org/releases/) + +# Other Useful OpenWRT Projects and Resources + +This section lists various other resources on the Internet which can be useful +with regard to installing or upgrading your OpenWRT router. + +* [uciparse · PyPI](https://pypi.org/project/uciparse/) + These tools were written to ease OpenWRT upgrades, making it easier to see + the differences between two config files. As of this writing (mid-2020), + OpenWRT upgrades often don't normalize upgraded config files in the same + way from version to version + +## Raspberry Pi 4 Network Performance Tuning + + * [(15) BIFRÖST Part 2 - RPi 4B Hybrid Bridging Router - Tuning Linux Network for Gigabit Line Speed | LinkedIn](https://www.linkedin.com/pulse/bifr%C3%B6st-rpi-4b-hybrid-routing-bridge-tuning-linux-network-corner/?trk=read_related_article-card_title) + very detailed article about tuning RPi4 network performance using IRQ + CPU pinning and other tricks. Kind of a hardwired ues of `irqbalance` + + +## Post Install Actions + +* [[OpenWrt Wiki] UCI defaults](https://openwrt.org/docs/guide-developer/uci-defaults) + describes how to setup some defaults at first boot +* [[OpenWrt Wiki] Opkg extras](https://openwrt.org/docs/guide-user/advanced/opkg_extras) + provides a way to reinstall user-installed packages after an upgrade +* [[OpenWrt Wiki] UCI extras](https://openwrt.org/docs/guide-user/advanced/uci_extras) + describes how to add files to `/etc/uci-defaults` which can configure + your OpenWRT install after an initial install or upgrade. +* [[OpenWrt Wiki] Opkg extras](https://openwrt.org/docs/guide-user/advanced/opkg_extras) + extends `opkg` to — among other things — handle user-installed packages +* [How to automate upgrade? - For Developers - OpenWrt Forum](https://forum.openwrt.org/t/how-to-automate-upgrade/72636/12) + has some pointers on how to re-install user installed packages when updating. + +## OpenWRT Remote Control Projects + + * [jumpscale7/openwrt-remote-manager: GitHub](https://github.com/jumpscale7/openwrt-remote-manager) + A Python module for remotely managing an OpenWRT instance. 22 stars, 6 forks, last active 2015. + Requires package `luci-mod-rpc` installed on your OpenWRT + * [fbradyirl/openwrt-luci-rpc](https://github.com/fbradyirl/openwrt-luci-rpc) + * Good docs, 17 stars, 10 forks, last active 2021-Mar + * Only an RPC transport. All functions must be implemented on top of that. + * [openwrt-luci-rpc · PyPI](https://pypi.org/project/openwrt-luci-rpc/) + * [Noltari/python-ubus-rpc](https://github.com/Noltari/python-ubus-rpc). + * Python module implementing an interface to the OpenWrt ubus RPC API. + * Looks pretty sweet. 0 stars, 0 forks, 2021-feb-17. + * Limited number of functions implemented. + * [openwrt-ubus-rpc · PyPI](https://pypi.org/project/openwrt-ubus-rpc/) + * [jianingy/openwrt-wac: GitHub](https://github.com/jianingy/openwrt-wac) + A simple command line tool for managing groups of OpenWRT based routers. + * 3 stars, 1 fork, last active 2016 + * [openwrt-wac · PyPI](https://pypi.org/project/openwrt-wac/) + + +## Some Useful OpenWRT Settings + +Disable DHCP server on LAN: `uci set dhcp.lan.ignore=1 && uci commit && /etc/init.d/dnsmasq restart` + +# Similar projects + + * [Github ansemjo/openwrtbuilder](https://gist.github.com/ansemjo/cb41677a76f1c063527744438b03b932) + *OpenWRT builder - using the imagebuilder to compile a custom image* + is a small, simple shell script that does pretty much the same as this script + although with command-line arguments instead of a configuration file. + +---- +Convert this markdown doc to HTML with pandoc: + +pandoc --toc --self-contained -t html -o README.html README.md diff --git a/configs/archer-c7-v2 b/configs/archer-c7-v2 new file mode 100644 index 0000000..fca5358 --- /dev/null +++ b/configs/archer-c7-v2 @@ -0,0 +1,47 @@ +#!/bin/sh +# Archer C7 v2 configuration for openwrt-image-buildomatic + +# Running OpenWRT image builder 18.06 on Ubuntu 20.04 requires at least: +# sudo apt install python2 libncurses5-dev + +#RELEASE=${RELEASE_18_06} +#RELEASE=${RELEASE_19_07} +RELEASE=${RELEASE_21_02} + +if [ -z "${RELEASE}" ]; then echo "RELEASE is empty"; return 1; fi + +# See targets here: https://downloads.openwrt.org/snapshots/targets +# +# device can be built with one of two TARGETs: +# ar71xx/generic is usable for releases 17.01, 18.06, 19.07 +# ath79/generic is usable for releases 19.07, 21.02 +if echo "${RELEASE}" | grep -q "18.06"; then + TARGET=ar71xx/generic + PROFILE="archer-c7-v2" +else + TARGET=ath79/generic + PROFILE="tplink_archer-c7-v2" +fi + +# This package list works for releases 18.06, 19.07 aND 21.02 +PACKAGES=" +luci +luci-theme-bootstrap luci-theme-openwrt +luci-app-uhttpd +luci-app-wireguard kmod-wireguard wireguard-tools qrencode +luci-app-fwknopd fwknopd +luci-app-sqm sqm-scripts sqm-scripts-extra +luci-app-https-dns-proxy https-dns-proxy +luci-app-ddns +luci-app-statistics + collectd collectd-mod-cpu collectd-mod-irq collectd-mod-thermal + collectd-mod-network collectd-mod-iwinfo collectd-mod-ping + collectd-mod-processes collectd-mod-exec +iptables-mod-geoip +-ip-tiny ip-full +zoneinfo-europe +" + +FILES="" + +DISABLED_SERVICES="" diff --git a/configs/common b/configs/common new file mode 100644 index 0000000..657ac8c --- /dev/null +++ b/configs/common @@ -0,0 +1,56 @@ +#!/bin/sh +# Common configuration include for openwrt-image-buildomatic +# This file is ALWAYS read before the device specific file +# so the device config file can use any of the variables here +# +# List of all OpenWRT packages: https://openwrt.org/packages/start + +# Latest point release of each main release +RELEASE_18_06="18.06.9" +RELEASE_19_07="19.07.7" +RELEASE_21_02="21.02.0-rc1" + +# Popular Luci apps +# This list is probably only valid for the 21.02 release +# +# luci-mod-rpc is for remote control, see https://openwrt.org/docs/techref/ubus +packages_luci_apps=" +luci +luci-theme-openwrt-2020 luci-theme-bootstrap luci-theme-openwrt +luci-app-uhttpd +luci-app-commands +luci-app-wireguard kmod-wireguard wireguard-tools qrencode +luci-app-acme acme ca-certificates +luci-app-adblock adblock +luci-app-fwknopd fwknopd +luci-app-banip banip +luci-app-sqm sqm-scripts sqm-scripts-extra +luci-app-https-dns-proxy https-dns-proxy +luci-app-ddns +luci-app-statistics + collectd collectd-mod-cpu collectd-mod-irq collectd-mod-thermal + collectd-mod-network collectd-mod-sqm collectd-mod-iwinfo collectd-mod-ping + collectd-mod-ethstat + collectd-mod-processes collectd-mod-exec +luci-mod-rpc +" + +# luci-app-nlbwmon nlbwmon +# luci-app-dnscrypt-proxy +# luci-theme-material has intentionally been left out (it is rather ugly) + +# All the most common USB Ethernet adapter kernel drivers +packages_usb_eth=" +kmod-usb-core kmod-usb-net kmod-mii +kmod-usb-net-asix kmod-usb-net-asix-ax88179 kmod-usb-net-cdc-ether kmod-usb-net-ipheth +kmod-usb-net-kaweth kmod-usb-net-mcs7830 kmod-usb-net-pegasus kmod-usb-net-rtl8150 +kmod-usb-net-rtl8152 kmod-usb-net-sr9700 +" + +# USB storage auto mounting +packages_usb_storage=" +kmod-fs-exfat kmod-fs-ext4 kmod-fs-ntfs kmod-fuse kmod-usb-storage +kmod-usb2 kmod-usb2-pci kmod-usb3 kmod-fs-autofs4 kmod-usb-ohci kmod-usb-uhci +block-mount blockd e2fsprogs fdisk +" + diff --git a/configs/rpi-4 b/configs/rpi-4 new file mode 100644 index 0000000..0d537b2 --- /dev/null +++ b/configs/rpi-4 @@ -0,0 +1,44 @@ +#!/bin/sh +# Raspberry PI 4 configuration for openwrt-image-buildomatic + +# Which target CPU to build image for. +# See targets here: https://downloads.openwrt.org/snapshots/targets +TARGET=bcm27xx/bcm2711 + +# Currently there is only the "rpi-4" build profile for this TARGET +PROFILE="rpi-4" + +RELEASE="21.02.0-rc1" + +# See https://openwrt.org/packages/start +# +# - irqbalance package distributes hardware interrupts across +# processors on a multiprocessor system in order to increase performance. +# - bcm27xx-eeprom package provides rpi-eeprom-update, rpi-eeprom-config +# commands for the RPi-4 eeprom +# - netperf installs /usr/bin/netserver and is used by various speedtest scripts +# See https://forum.openwrt.org/t/package-to-run-speed-test-on-router/16016 +PACKAGES=" +${packages_usb_eth} +${packages_luci_apps} +${packages_usb_storage} +bcm27xx-eeprom +irqbalance +ethtool +netperf speedtest-netperf iperf3 +nmap netdiscover arp-scan arp-scan-database +iptables-mod-geoip +igmpproxy +-dnsmasq dnsmasq-full +-ip-tiny ip-full +zoneinfo-europe +procps-ng procps-ng-watch procps-ng-pgrep procps-ng-ps procps-ng-top procps-ng-free +curl wget rsync file htop lsof less mc tree usbutils bash +strace +openssh-sftp-server +" +# openssh-server +FILES="rpi-4-files" + +DISABLED_SERVICES="\ +adblock banip collectd ddns fwknopd https-dns-proxy igmpproxy netserver sqm" diff --git a/configs/tl-wdr3600-v1 b/configs/tl-wdr3600-v1 new file mode 100644 index 0000000..8ffd150 --- /dev/null +++ b/configs/tl-wdr3600-v1 @@ -0,0 +1,31 @@ +#!/bin/sh +# TP-Link WDR3600 v1 configuration for openwrt-image-buildomatic + +# Running OpenWRT image builder 18.06 on Ubuntu 20.04 requires at least: +# sudo apt install python2 libncurses5-dev + +#RELEASE=${RELEASE_18_06} +#RELEASE=${RELEASE_19_07} +RELEASE=${RELEASE_21_02} + +if [ -z "${RELEASE}" ]; then echo "RELEASE is empty"; return 1; fi + +# See targets here: https://downloads.openwrt.org/snapshots/targets +# +# device can be built with one of two TARGETs: +# ar71xx/generic is usable for releases 17.01, 18.06, 19.07 +# ath79/generic is usable for releases 19.07, 21.02 +if echo "${RELEASE}" | grep -q "18.06"; then + TARGET=ar71xx/generic + PROFILE="tl-wdr3600-v1" +else + TARGET=ath79/generic + PROFILE="tplink_tl-wdr3600-v1" +fi + +# See https://openwrt.org/packages/start +PACKAGES="luci htop" + +FILES="" + +DISABLED_SERVICES="" diff --git a/oi-build b/oi-build new file mode 100755 index 0000000..2420ba6 --- /dev/null +++ b/oi-build @@ -0,0 +1,292 @@ +#!/usr/bin/env bash +# Script to download and run OpenWRT image builder with some config file + + +############################################################################### +# OpenWRT target image config defaults +############################################################################### + +# Default configuration (file) to use +config_name="rpi-4" + +# Release to fetch and build (can be "snapshot") +RELEASE="" + +# Which target CPU architecture to build image for. +# See targets here: https://downloads.openwrt.org/snapshots/targets +TARGET="" + +# What profile (platform/device) to build for this TARGET +PROFILE="" + +FILES="" + +DISABLED_SERVICES="" + +# See https://openwrt.org/packages/ for list of all packages +PACKAGES=" +luci luci-app-uhttpd +luci-theme-bootstrap luci-theme-openwrt luci-theme-openwrt-2020 +luci-app-wireguard kmod-wireguard wireguard-tools +zoneinfo-europe +kmod-usb-net-rtl8150 kmod-usb-net-rtl8152 +" + +# glob to use for searching for USB SD-card reader device +dev_usb_sdcard_glob=/dev/disk/by-id/usb-Generic-*-0:0 + + +############################################################################### +# console colors +############################################################################### + +if [ -t 1 ]; then + c_red="\e[31m"; c_green="\e[32m"; c_yellow="\e[33m"; c_mag="\e[35m" + c_cyan="\e[36m"; c_white="\e[37m"; c_ired="\e[91m"; c_igreen="\e[92m" + c_iyellow="\e[93m"; c_imag="\e[95m"; c_icyan="\e[96m"; c_iwhite="\e[97m" + c_igrey="\e[90m"; c_reset="\e[0m"; c_bold="\e[1m"; c_inv="\e[7m" +fi +echo_run() { echo -e "\$ ${c_green}$*${c_reset}"; if [ -z "${dryrun}" ]; then eval "$*"; else true; fi; } +echo_run_nodry() { echo -e "\$ ${c_green}$*${c_reset}"; eval "$*"; } +echo_ok() { echo -e "${c_imag}$*${c_reset}"; } +echo_note() { echo -e "${c_iyellow}$*${c_reset}"; } +echo_bold() { echo -e "${c_icyan}$*${c_reset}"; } +echo_inv() { echo -e "${c_inv}$*${c_reset}"; } + +warn() { echo >&2 -e "${c_iyellow}WARNING: $*${c_reset}"; } +error() { echo >&2 -e "${c_ired}ERROR: $*${c_reset}"; } +die() { error "$*"; exit 1; } +verb() { [ -n "${VERBOSE}" ] && echo "$*"; } + +############################################################################### +# +############################################################################### + +openwrt_url=https://downloads.openwrt.org + +load_config() { + local name=$1 + . configs/common || die "FAILED to load config from ${f}" + for f in configs/${name} configs/${name}.sh; do + if [ -f ${f} ]; then + . "${f}" || die "FAILED to load config from ${f}" + echo "Loaded config ${f}" + return 0 + fi + done + die "configuration file ${name} not found in configs" +} + +# setup file and directory names of image builder and final image: +# target_with_dash +# imagebuilder_dir +# image_file_base +# image_tarball +init_vars() { + echo "Current config:" + for v in RELEASE TARGET PROFILE; do + printf " %-10s = %s\n" "${v}" "${!v}" + [ -n "${!v}" ] || die "Config variable ${v} has no value" + done + + target_with_dash=$(echo "${TARGET}" | tr '/' '-') + + if [ "${RELEASE}" = "snapshot" ]; then + tarball=openwrt-imagebuilder-${target_with_dash}.Linux-x86_64.tar.xz + url_path="snapshots/targets/${TARGET}/${tarball}" + else + tarball=openwrt-imagebuilder-${RELEASE}-${target_with_dash}.Linux-x86_64.tar.xz + url_path="releases/${RELEASE}/targets/${TARGET}/${tarball}" + fi + imagebuilder_dir=${tarball%%.tar*} +} + +fetch_image_builder() { + echo_note "Downloading ${RELEASE} build from ${url_path}" + + if [ -f "${tarball}" ]; then + echo_ok "${tarball} already exists, skipping fetch" + else + echo_run "wget ${openwrt_url}/${url_path}" || die "wget FAILED" + fi + + if [ -d ${imagebuilder_dir} ]; then + echo_ok "${imagebuilder_dir} already exists, skipping unpack" + else + echo_run "tar xf ${tarball}" || die "tar xf FAILED" + fi +} + +locate_files_dir() { + for d in ../configs .. .; do + local path=${d}/${FILES} + if [ -d ${path} ]; then + echo_note "Using FILES = ${path}" + FILES=${path} + return 0 + fi + done + die "FAILED to locate FILES directory: ${FILES}" +} + +image_build() { + echo_inv "make image" + + locate_files_dir + + PACKAGES=$(echo "${PACKAGES}" | tr '\n' ' ') + echo_run "make image PROFILE='${PROFILE}' FILES='${FILES}' PACKAGES='${PACKAGES}' DISABLED_SERVICES='${DISABLED_SERVICES}'" || + die "image build FAILED" +} + +image_builder_make_info() { + echo_inv "make info" + + echo_run "make info" || + die "image build FAILED" + + local file=bin/targets/${TARGET}/${image_tarball} + [ -f ${file} ] && + echo_run "gunzip -f ${file}" +} + +image_write_to_sdcard() { + echo_inv "Determining SD card write command" + + # find the USB device that holds the SD-Card: + if echo_run "ls -lh ${dev_usb_sdcard_glob}"; then + realdev=$(readlink -f ${dev_usb_sdcard_glob}) + echo_note "Using device ${realdev}" + else + echo_note "No USB storage device detected:" + echo_note "You will have to insert correct USB_STORAGE device in the command below" + realdev="USB_STORAGE" + fi + + image_names_gz=$(find bin/targets/${TARGET} -name '*factory.img.gz') + [ -n "${image_names_gz}" ] && + echo "Unzipping images" + for f in ${image_names_gz}; do + echo_run "gunzip -f ${f}" + done + + echo "Finding factory images for SD card writing:" + echo_run "(cd bin/targets/${TARGET} && ls -1 *factory.img)" + image_names=$(find bin/targets/${TARGET} -name '*factory.img') + image_tarball=$(echo "${image_names}" | head -n1) + + echo_note "Here are the commands for SD card write. Verify before running:" + if mount | grep "${realdev}" >/dev/null; then + echo_bold "sudo umount ${realdev}?" + fi + echo_bold "sudo dd if=${imagebuilder_dir}/${image_tarball} \\ + of=${realdev} \\ + bs=1M status=progress && sync" + [ "${realdev}" = "USB_STORAGE" ] && \ + echo "where USB_STORAGE is the USB storage device" +} + +############################################################################### +# +############################################################################### + +usage() { + cat </dev/null; then + cd .. + echo_note "Original and modified rootfs is here:" + echo_run "ls -d1 ${imagebuilder_dir}/build_dir/target-*/root*" + cd - >/dev/null +fi + +if [ -d bin/targets/${TARGET} ]; then + echo_run "ls -lh bin/targets/${TARGET}/" + echo_note "Final images are in ${imagebuilder_dir}/bin/targets/${TARGET}" +fi