From 8d4c6bc6cf9cd29fd93af1e5ec861e9f290a5877 Mon Sep 17 00:00:00 2001 From: Eric Hameleers Date: Sat, 1 Sep 2018 12:45:59 +0200 Subject: Finish the zstd compression implementation in liveslak The new zstd compression support for squashfs modules allows for a much snappier system because zstd decompression is 5 times faster than that of xz (see https://sourceforge.net/p/squashfs/mailman/message/35989805/), but at the expense of ISO sizes that are at least 10% bigger. The latter means that we have to do some more pruning of the XFCE and PLASMA5 images to make these fit in a CDROM and DVD image, respectively. Here are some initial benchmarks with PLASMA5 Live ISO images, booting in QEMU (times are in minutes:seconds): Stage ZSTD XZ --------------------------------------- Boot to runlevel 4 start 0:39 0:50 SDDM Login Manager visible 0:55 1:26 Plasma5 desktop ready 1:42 3:17 I also tested two of the bigger applications, measuring the time between entering the command in a terminal and having a usable application window: Stage ZSTD XZ ---------------------------------------- LO Writer window visible 0:17 0:34 Chromium window visible 0:09 0:14 It is obvious that a big Desktop Environment like Plasma5 where a lot of binaries have to be loaded from their squashfs modules benefits a lot from zstd, because after booting, the Plasma5 DE is available in roughly 53% of the time it takes when using xz compression. Zstd support in squashfs was added to Linux kernel 4.14. That means, no customimzation of Slackware is required to make the Live OS work from zstd-compressed squashfs modules. In order to *create* these zstd-compressed squashfs modules, you will need some custom packages at the moment, until they get added to Slackware-current: zstd and a rebuild squashfs-tools to add the lacking zstd support to mksqhashfs/unsquashfs. --- liveinit.tpl | 27 +++++++++++++++++++++---- make_slackware_live.sh | 55 ++++++++++++++++++++++++++++++++------------------ 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/liveinit.tpl b/liveinit.tpl index da1b4ff..e4f38d1 100755 --- a/liveinit.tpl +++ b/liveinit.tpl @@ -46,6 +46,8 @@ VERSION="@VERSION@" LIVEUID="@LIVEUID@" +SQ_EXT_AVAIL="@SQ_EXT_AVAIL@" + LIVEMEDIA="" LIVEPATH="" @@ -479,6 +481,21 @@ if [ "$RESCUE" = "" ]; then echo "$lodev" } + mod_base() { + MY_MOD="$1" + + echo $(basename ${MY_MOD}) |rev |cut -d. -f2- |rev + } + + find_mod() { + MY_LOC="$1" + + ( for MY_EXT in ${SQ_EXT_AVAIL} ; do + echo "$(find ${MY_LOC} -name "*.${MY_EXT}" 2>/dev/null)" + done + ) | sort + } + find_modloc() { MY_LOC="$1" MY_BASE="$2" @@ -501,8 +518,10 @@ if [ "$RESCUE" = "" ]; then # SUBSYS can be 'system', 'addons', 'optional': SUBSYS="$1" - for MODULE in $(find /mnt/media/${LIVEMAIN}/${SUBSYS}/ -name "*.sxz" 2>/dev/null |sort) ; do - MODBASE="$(basename ${MODULE} .sxz)" + # Find all supported modules: + for MODULE in $(find_mod /mnt/media/${LIVEMAIN}/${SUBSYS}/) ; do + # Strip path and extension from the modulename: + MODBASE="$(mod_base ${MODULE})" if [ "$SUBSYS" = "optional" ]; then # Load one or more optionals by using boot parameter 'load': # load=mod1[,mod2[,mod3]] @@ -715,7 +734,7 @@ if [ "$RESCUE" = "" ]; then # Start assembling our live system components below /mnt/live : mkdir /mnt/live - # Mount our squashed modules (.sxz extension). + # Mount our squashed modules (.sxz or other supported extension) mkdir /mnt/live/modules if [ $TORAM -ne 0 ]; then @@ -1192,7 +1211,7 @@ EOT RUN_DEPMOD=0 for MOD in $(cat /sys/block/loop*/loop/backing_file |grep -E "optional|addons") do - if [ -d /mnt/live/modules/$(basename $MOD .sxz)/lib/modules/$(uname -r)/ ] + if [ -d /mnt/live/modules/$(mod_base $MOD)/lib/modules/$(uname -r)/ ] then # Found kernel modules directory; we need to make a 'depmod' call. RUN_DEPMOD=1 diff --git a/make_slackware_live.sh b/make_slackware_live.sh index 26645c3..e035c3d 100755 --- a/make_slackware_live.sh +++ b/make_slackware_live.sh @@ -228,42 +228,29 @@ ADD_CACERT=${ADD_CACERT:-"YES"} COMPR=${COMPR:-"xz --check=crc32"} # What compressors are available? -SQ_COMP_AVAIL=(gzip lzma lzo xz zstd) +SQ_COMP_AVAIL="gzip lzma lzo xz zstd" + +# What module exttensions do we accept: +SQ_EXT_AVAIL="sxz sfz szs xzm" # Compressor optimizations: declare -A SQ_COMP_PARAMS_DEF SQ_COMP_PARAMS_DEF[gzip]="" SQ_COMP_PARAMS_DEF[lzma]="" SQ_COMP_PARAMS_DEF[lzo]="" -SQ_COMP_PARAMS_DEF[xz]=""-b 512k -Xdict-size 100%" +SQ_COMP_PARAMS_DEF[xz]="-b 512k -Xdict-size 100%" SQ_COMP_PARAMS_DEF[zstd]="-b 512k -Xcompression-level 16" declare -A SQ_COMP_PARAMS_OPT SQ_COMP_PARAMS_OPT[gzip]="" SQ_COMP_PARAMS_OPT[lzma]="" SQ_COMP_PARAMS_OPT[lzo]="" -SQ_COMP_PARAMS_OPT[xz]=""-b 1M" +SQ_COMP_PARAMS_OPT[xz]="-b 1M" SQ_COMP_PARAMS_OPT[zstd]="-b 1M -Xcompression-level 19" # What compression to use for the squashfs modules? # Default is xz, alternatives are gzip, lzma, lzo, zstd: SQ_COMP=${SQ_COMP:-"xz"} -# Test whether the compressor of choice is supported by the script: -if [[ ${SQ_COMP_AVAIL[*]} =~ ${SQ_COMP} ]]; then - echo "*** Compressor '${SQ_COMP}' not supported by $(basename $0 .sh)!" - echo "*** Select one of '${SQ_COMP_AVAIL[@]}'" - exit 1 -fi - -# What compression parameters to use? -# For our lean XFCE image we try to achieve max compression, -# at the expense of runtime latency: -if [ "$LIVEDE" = "XFCE" ] ; then - SQ_COMP_PARAMS=${SQ_COMP_PARAMS:-"${SQ_COMP_PARAMS_OPT[${SQ_COMP}]}"} -else - SQ_COMP_PARAMS=${SQ_COMP_PARAMS:-"${SQ_COMP_PARAMS_DEF[${SQ_COMP}]}"} -fi - # Mount point where Live filesystem is assembled (no storage requirements): LIVE_ROOTDIR=${LIVE_ROOTDIR:-"/mnt/slackwarelive"} @@ -863,7 +850,7 @@ create_iso() { # Action! # --------------------------------------------------------------------------- -while getopts "a:d:efhm:r:s:t:vz:GH:MO:R:X" Option +while getopts "a:c:d:efhm:r:s:t:vz:GH:MO:R:X" Option do case $Option in h ) @@ -883,6 +870,8 @@ do echo " -h This help." echo " -a arch Machine architecture (default: ${SL_ARCH})." echo " Use i586 for a 32bit ISO, x86_64 for 64bit." + echo " -c comp Squashfs compression (default: ${SQ_COMP})." + echo " Can be any of '${SQ_COMP_AVAIL}'." echo " -d desktoptype SLACKWARE (full Slack), KDE4 basic," echo " XFCE basic, PLASMA5, MATE, CINNAMON, DLACK." echo " -e Use ISO boot-load-size of 32 for computers." @@ -905,6 +894,8 @@ do ;; a ) SL_ARCH="${OPTARG}" ;; + c ) SQ_COMP="${OPTARG}" + ;; d ) LIVEDE="$(echo ${OPTARG} |tr a-z A-Z)" ;; e ) BOOTLOADSIZE=32 @@ -1012,6 +1003,29 @@ if [ ! -z "$PROG_MISSING" ] ; then exit 1 fi +# Test whether the compressor of choice is supported by the script: +if ! echo ${SQ_COMP_AVAIL} | grep -wq ${SQ_COMP} ; then + echo "-- Compressor '${SQ_COMP}' not supported by liveslak!" + echo "-- Select one of '${SQ_COMP_AVAIL}'" + exit 1 +else + # Test whether the local squashfs-tools support the compressor: + if ! mksquashfs 2>&1 | grep -Ewq "^[[:space:]]*${SQ_COMP}" ; then + echo "-- Compressor '${SQ_COMP}' not supported by your 'mksquashfs'!" + echo "-- Select another one from '${SQ_COMP_AVAIL}'" + exit 1 + fi +fi + +# What compression parameters to use? +# For our lean XFCE image we try to achieve max compression, +# at the expense of runtime latency: +if [ "$LIVEDE" = "XFCE" ] ; then + SQ_COMP_PARAMS=${SQ_COMP_PARAMS:-"${SQ_COMP_PARAMS_OPT[${SQ_COMP}]}"} +else + SQ_COMP_PARAMS=${SQ_COMP_PARAMS:-"${SQ_COMP_PARAMS_DEF[${SQ_COMP}]}"} +fi + # Check rsync progress report capability: if [ -z "$(rsync --info=progress2 2>&1 |grep "unknown option")" ]; then # Use recent rsync to display some progress: @@ -2268,6 +2282,7 @@ cat $LIVE_TOOLDIR/liveinit.tpl | sed \ -e "s/@CDISTRO@/${DISTRO^}/g" \ -e "s/@UDISTRO@/${DISTRO^^}/g" \ -e "s/@VERSION@/$VERSION/g" \ + -e "s/@SQ_EXT_AVAIL@/${SQ_EXT_AVAIL}/" \ > ${LIVE_ROOTDIR}/boot/initrd-tree/init cat /dev/null > ${LIVE_ROOTDIR}/boot/initrd-tree/luksdev # We do not add openobex to the initrd and don't want to see irrelevant errors: -- cgit v1.2.3