summaryrefslogtreecommitdiffstats
path: root/source/make_world.sh
blob: efb52fb2e82691d6223011626e1ba140616bf7b9 (about) (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
#!/bin/bash
# Copyright 2018  Patrick J. Volkerding, Sebeka, Minnesota, USA
# All rights reserved.
#
# Redistribution and use of this script, with or without modification, is
# permitted provided that the following conditions are met:
#
# 1. Redistributions of this script must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#
#  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
#  WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
#  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO
#  EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
#  OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
#  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
#  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
#  ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# make_world for Slackware: rebuilds all the SlackBuild scripts that are
# listed in the given build list.
# Each line needs to look like (for example, and without beginning with "# "):
# a/grep/grep.SlackBuild
# For x11/KDE packages, you may specify a specific package to be built using
# the arguments understood by the SlackBuild. For example:
# kde/kde.SlackBuild kdelibs:kdelibs
# Any line beginning with "#" will be skipped.

# WARNING: This script has the potential to mess up your system.
# It is not recommended to run this on a production machine.
# This script is meant to be used on a fully installed and updated system.
# Using it on a partially installed system may result in broken packages,
# packages with missing features, or build failures. If there are circular
# dependencies, more than one build may be needed to get a correct package.
#
# Slackware is not Gentoo.
# For Amusement Purposes Only.

# Not all SlackBuilds are compatible with this script. In order for a build
# script to work, it needs these features:
# 1) cd into the script directory when run i.e.: cd $(dirname $0) ; CWD=$(pwd)
# 2) Handle $TMP properly (less of an issue if you don't change $TMP)
# 3) Support output package name reporting with PRINT_PACKAGE_NAME=yes

cd $(dirname $0) ; CWD=$(pwd)
# Some SlackBuilds do not work (yet) with a different $TMP setting.
TMP=${TMP:-/tmp}
export TMP
# Where the SlackBuild script puts the built package (i.e., $TMP).
OUTPUT_LOCATION=${OUTPUT_LOCATION:-$TMP}
# Used for logs and lock files.
LOGDIR=$TMP/make_world
# Lockfiles. You might need to purge these before restarting a build.
mkdir -p $LOGDIR/lock

# Set a custom Slackware source directory. By default we assume we are already
# in the source directory.
SLACKWARE_SOURCE_DIRECTORY=${SLACKWARE_SOURCE_DIRECTORY:-}
if [ ! -z "$SLACKWARE_SOURCE_DIRECTORY" ]; then
  # Make sure this ends in '/':
  if [ ! "$(echo $SLACKWARE_SOURCE_DIRECTORY | rev | cut -b 1)" = "/" ]; then
    SLACKWARE_SOURCE_DIRECTORY="${SLACKWARE_SOURCE_DIRECTORY}/"
  fi
fi 

# To wipe build directories and package creation directories after each
# package is built, set this to anything other than "no". You might need
# to use this if you're short on build space. NOTE: if you use this
# feature, you can NOT run more than one copy of this script at the same
# time! It will wipe build trees for other packages before they can finish.
# Otherwise, file locking is used and you may run as many parallel copies
# of this script as you think will help to speed things along.
WIPE_AFTER_BUILD=${WIPE_AFTER_BUILD:-no}

# NOTE: In case kde.SlackBuild or x11.SlackBuild are used to build everything
# in one shot, it's safer to just let stuff be installed twice...
#
## This variable is used to tell x11.SlackBuild not to use upgradepkg on the
## built package since this script already does that:
#UPGRADE_PACKAGES=${UPGRADE_PACKAGES:-"no"}
#export UPGRADE_PACKAGES
#
## This variable is used to tell kde.SlackBuild not to use upgradepkg on the
## built package since this script already does that:
#UPGRADE=${UPGRADE:-"no"}
#export UPGRADE

# Be kind, don't hit control-c! If you do, you might leave broken packages,
# logfiles, and locks in $TMP that will cause problems for you later. If you're
# not in a huge hurry to quit, create this file (replace with $TMP if needed):
# /tmp/make_world/lock/abort
# This will cause all instances of make_world.sh to exit when they complete the
# task they are working on.
rm -f $LOGDIR/lock/abort

BUILDLIST=${BUILDLIST:-$LOGDIR/buildlist}
if [ ! -r $BUILDLIST -a ! -r ${BUILDLIST}.lock ]; then
  # The buildlist does not exist, so attempt to create one that builds
  # everything except for the kernels (the kernel scripts are not compatible
  # with make_world.sh, mostly because of the need to reboot the new kernel).
  touch ${BUILDLIST} ${BUILDLIST}.lock
  echo "Generating list of packages to build in ${BUILDLIST}..."
  for script in ${SLACKWARE_SOURCE_DIRECTORY}*/*/*.SlackBuild ; do
    # Only add the script if the SlackBuild name matches the directory name:
    if [ "$(basename $(echo $script | cut -f 1 -d ' ') .SlackBuild)" = "$(echo $(dirname $(echo $script | cut -f 1 -d ' ')) | rev | cut -f 1 -d / | rev)" ]; then
      if [ "$uname -m)" = "x86_64" -a "$(basename $(echo $script | cut -f 1 -d ' '))" = "isapnptools.SlackBuild" ]; then
        # Don't try to build isapnptools on x86_64.
        continue
      fi
      echo $script >> $BUILDLIST
    fi
  done
  if [ -r ${SLACKWARE_SOURCE_DIRECTORY}kde/kde.SlackBuild ]; then
    echo "${SLACKWARE_SOURCE_DIRECTORY}kde/kde.SlackBuild" >> $BUILDLIST
  fi
  rm -f ${BUILDLIST}.lock
  # Set GEN_LIST_ONLY=yes if you'd like to exit after generating a build list.
  # You might want to do this to comment some build scripts out first, or if
  # you'd like to sort it into a "magic build order". ;-) This script is pretty
  # good at just brute-forcing things, though (with a few runs).
  if [ "$GEN_LIST_ONLY" = "yes" ]; then
    echo "Generated ${BUILDLIST}. Exiting."
    exit 0
  fi
fi
if [ -r ${BUILDLIST}.lock ]; then
  echo -n "Waiting for ${BUILDLIST}.lock to be removed..."
  while [ 0 ]; do
    if [ ! -r ${BUILDLIST}.lock ]; then
      break
    fi
    sleep 5
  done
  echo " done."
fi
echo "Using buildlist $BUILDLIST."

# To use shuffle mode (build packages in a random order each time through),
# pass SHUFFLE=yes (or anything other than "no") to this script.
SHUFFLE=${SHUFFLE:-no}
if [ "$SHUFFLE" = "no" ]; then
  SHUF=cat
else
  SHUF=shuf
fi

# To keep repeating the build list, set $REPEAT to anything other than "no":
REPEAT=${REPEAT:-no}

# To always rebuild a SlackBuild even if already built packages are found, set
# FORCE_BUILD=yes:
FORCE_BUILD=${FORCE_BUILD:-no}

# Function to do the build:
do_build() {
  if [ "$HAVE_GLOBAL_LOCK" = "true" ]; then
    # Wait for other builds to complete
    echo -n "have global lock, waiting for other builds to complete... "
    while [ 0 ]; do
      sleep 5
      if ! /bin/ls $LOGDIR/lock/*.lock 1> /dev/null 2> /dev/null ; then
        echo -n "done, continuing... "
        break
      fi
    done
  fi
  # If we're trying again, we don't want to leave a failure log in the logs
  # directory. But save it just in case...
  if [ -r $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed ]; then
    mkdir -p $LOGDIR/faillog-backups
    mv $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed $LOGDIR/faillog-backups
  fi
  $buildscript &> $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log
  if [ ! $? = 0 ]; then
    # Exit code from SlackBuild indicated an error:
    echo "$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) failed to build."
    mv $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed
  elif [ ! -r $OUTPUT_LOCATION/$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) ]; then
    # No error code returned from SlackBuild, but the package(s) were not found.
    # Possibly the SlackBuild doesn't honor $TMP, and a non-/tmp $TMP variable was set?
    echo "$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) failed to build."
    mv $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log $LOGDIR/$(basename $(echo $buildscript | cut -f 1 -d ' ')).log.failed
  else
    # Figure out a progress report to include with the successful build message:
    cat $BUILDLIST | grep -v "^$" | grep -v "^#" | sort | uniq | cut -f 1 -d ' ' | rev | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-to-build.$$
    # OK, we don't know if every *.log is actually finished if we're running
    # more than one make_world.sh, but whatever. It's an estimate.
    /bin/ls $LOGDIR/*.log | rev | cut -f 2,3 -d . | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-built-or-building.$$
    NUMTOTAL="$(cat $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    NUMBUILT="$(grep -x -f $LOGDIR/tmp-pkgs-built-or-building.$$ $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    rm -f $LOGDIR/tmp-pkgs-to-build.$$ $LOGDIR/tmp-pkgs-built-or-building.$$
    echo "$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) built successfully ($NUMBUILT/$NUMTOTAL)."
    for package in $(PRINT_PACKAGE_NAME=foo $buildscript) ; do
      upgradepkg --install-new --reinstall $OUTPUT_LOCATION/$package > /dev/null 2>&1
    done
    # The post-build cleanup is pretty sloppy. It will not clean up all of
    # the build-related residue, and it may possibly delete some things that
    # this script did not put there. It's also not compatible with running
    # more than one copy of make_world.sh simultaneously.
    # Think more than twice before using this.
    if [ ! "$WIPE_AFTER_BUILD" = "no" ]; then
      ( cd $TMP
        rm -f configure CMakeLists.txt
        for findconfigure in */configure ; do
          rm -rf "$(dirname $findconfigure)"
        done
        for findautogen in */autogen.sh ; do
          rm -rf "$(dirname $findautogen)"
        done
        for findcmake in */CMakeLists.txt ; do
          rm -rf "$(dirname $findcmake)"
        done
        for findmake in */Makefile ; do
          rm -rf "$(dirname $findmake)"
        done
        for findmeson in */meson.build ; do
          rm -rf "$(dirname $findmeson)"
        done
        for findpython in */setup.py ; do
          rm -rf "$(dirname $findpython)"
        done
        rm -rf package-*
      )
    fi
  fi
}

# Main loop:
while [ 0 ]; do
  # Skip any blank lines or lines in the buildlist that begin with #:
  cat $BUILDLIST | grep -v "^$" | grep -v "^#" | $SHUF | while read buildscript ; do
    if [ -r $LOGDIR/lock/abort ]; then
      echo "Exiting (abort requested)."
      exit 0
    fi
    # If there's a global lock, we have to wait for it to be released:
    if [ -r $LOGDIR/lock/lock.global ]; then
      HELD_BY="$(cat $LOGDIR/lock/lock.global)"
      echo -n "Waiting for global lock release (held by ${HELD_BY})... "
      while [ 0 ]; do
        sleep 10
        if [ ! -r $LOGDIR/lock/lock.global ]; then
          echo "released."
          break
        fi
        if [ ! "${HELD_BY}" = "$(cat $LOGDIR/lock/lock.global)" ]; then
          HELD_BY="$(cat $LOGDIR/lock/lock.global)"
          echo
          echo -n "Waiting for global lock release (held by ${HELD_BY})... "
        fi
      done
    fi
    echo -n "Working on $buildscript... "
    if [ -r $OUTPUT_LOCATION/$(PRINT_PACKAGE_NAME=foo $buildscript | head -n 1) -a $FORCE_BUILD = no ]; then
      echo "skipping because built package(s) were found."
      continue
    fi
    # Use flock to only allow one instance of this script to work on a given
    # SlackBuild script at a time. If the SlackBuild is already locked, we'll
    # just move on to the next one. Yes, you can run more than one copy of this
    # script at the same time to speed things up!
    #
    # See if we need a global lock. Some SlackBuilds are disruptive and other
    # builds should not take place until they have completed. For example, perl
    # removes itself from the system during the build. Assume that we need a
    # global lock for any package that uses removepkg, upgradepkg, slacktrack,
    # or trackbuild. Also, you may add the identifier REQUIRE_GLOBAL_LOCK
    # anywhere in a SlackBuild script to make it require the global lock.
    HAVE_GLOBAL_LOCK=false
    if grep -q -e removepkg -e upgradepkg -e slacktrack -e trackbuild -e REQUIRE_GLOBAL_LOCK $(dirname $(echo $buildscript | cut -f 1 -d ' '))/$(basename $(echo $buildscript | cut -f 1 -d ' ')) ; then
      # pkgtools, x11, and KDE all trigger the detection above, but none of them
      # really need the global lock. So only request the lock if the build
      # script is not one of those.
      if ! echo "$(basename $(echo $buildscript | cut -f 1 -d ' '))" | grep -q -x -e pkgtools.SlackBuild -e x11.SlackBuild -e kde.SlackBuild ; then
        HAVE_GLOBAL_LOCK=true
      fi
    fi
    if [ "$HAVE_GLOBAL_LOCK" = "true" ]; then
      ( flock 9 || exit 11
        echo $(basename $(echo $buildscript | cut -f 1 -d ' ')) >> $LOGDIR/lock/lock.global
        do_build
      ) 9> $LOGDIR/lock/lock.global
      # Remove lock file:
      rm -f $LOGDIR/lock/lock.global
    else
      ( flock -n 9 || exit 11
        do_build
      ) 9> $LOGDIR/lock/$(basename $(echo $buildscript | cut -f 1 -d ' ')).lock
      if [ $? = 11 ]; then
        echo "skipping (locked by another make_world.sh instance)."
        continue
      fi
      # Remove lock file:
      rm -f $LOGDIR/lock/$(basename $(echo $buildscript | cut -f 1 -d ' ')).lock
    fi
  done
  if [ "$REPEAT" = "no" ]; then
    break
  else
    # Figure out if we skipped everything and exit REPEAT mode if we did:
    cat $BUILDLIST | grep -v "^$" | grep -v "^#" | sort | uniq | cut -f 1 -d ' ' | rev | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-to-build.$$
    /bin/ls $LOGDIR/*.log | rev | cut -f 2,3 -d . | cut -f 1 -d / | rev > $LOGDIR/tmp-pkgs-built-or-building.$$
    NUMTOTAL="$(cat $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    NUMBUILT="$(grep -x -f $LOGDIR/tmp-pkgs-built-or-building.$$ $LOGDIR/tmp-pkgs-to-build.$$ | wc -l)"
    rm -f $LOGDIR/tmp-pkgs-to-build.$$ $LOGDIR/tmp-pkgs-built-or-building.$$
    if [ "$NUMTOTAL" = "$NUMBUILT" ]; then
      echo "All packages have been built ($NUMBUILT/$NUMTOTAL). Exiting."
      break
    else
      echo "Repeating BUILDLIST since some packages are not built yet ($NUMBUILT/$NUMTOTAL complete)."
    fi
  fi
done