#!/bin/bash
# texmf_get.sh (c) 2016-2017 Johannes Schoepfer, Germany, slackbuilds[at]schoepfer[dot]info
# All rights reserved.
#
# Redistribution and use of this script, with or without modification, is
# permitted provided that the following conditions are met:
#
# 1. Redistributions of this script must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# V 0.14.2
#
# Prepare xz-compressed tarballs of texlive-texmf-trees based on texlive.tlpdb
# This script takes care of dependencies(as far as these are present in texlive.tlpdb) of collections and packages,
# and that every texlive-package is included only once.
# The editions(base/extra/fonts/docs) should contain nobinaries(exception biber)
# -base: the most usefull stuff, all metafonts, all manpages, the most
# binaries/scripts, 65mb 2017-11-07
# -docs: -base documentation only, no manpages/GNU infofiles
# -fonts: fonts only, no metafonts
# -extra: remaining stuff
#
# texlive netarchive policy: Every package is included as dependency
# in exactly one collection. A package may have dependencies on other
# packages from any collection.
# package source: http://mirror.ctan.org/systems/texlive/tlnet/archive/
#set -e
MAJORVERSION=2017
# globally excluded packages, which e.g. are
# -useless without tlmgr-installer
# -non-linux
# -covered by an external package
# -obsolete
global_exclude="
asymptote
tlcockpit
tlshell
texosquery
aleph
omega
antomega
omegaware
lambda
otibet
cslatex
"
texmf_editions () {
# At first, $corepackages for the base, and other essentials
PACKAGES="
$(cat $corepackages)
collection-basic
collection-latex
collection-metapost
collection-plaingeneric
collection-fontutils
collection-luatex
collection-context
cbfonts-fd
xetex-devanagari
" texmfget base || exit 1
# Now the fonts package to make sure that big fonts like cm-super are not included elsewhere as dependency
PACKAGES="
cm-super
cbfonts
ipaex
wadalab
ethiop-t1
fonts-tlwg
uhc
fandol
arphic
arphic-ttf
nanumtype1
baekmuk
unfonts-extra
unfonts-core
" texmfget fonts || exit 1
# put some stuff in "extra" to before these make their way into "base" as dependency
PACKAGES="
$(grep ^"name biblatex" $db | grep -v '\.' | cut -d' ' -f2 )
biber.x86_64-linux
biber.i386-linux
bib2gls
bibarts
arara
latex2nemeth
ghsystem
adobemapping
knitting
pgfornament
pgfplots
arabi
nwejm
uantwerpendocs
sduthesis
stellenbosch
fithesis
gregoriotex
lilyglyphs
musixtex-fonts
beebe
velthuis
mwe
pdfx
media9
pst-cox
pst-poker
pst-vectorian
pst-geo
quran
ijsra
fibeamer
udesoftec
xduthesis
hustthesis
bangorcsthesis
sapthesis
uowthesis
cs
pl
cc-pl
tipa
kerkis
amiri
cns
vntex
montex
xcharter
fonts-churchslavonic
japanese-otf
sanskrit-t1
skaknew
padauk
" texmfget extra || exit 1
# Completing the base
PACKAGES="
collection-langcyrillic
collection-langczechslovak
collection-langeuropean
collection-langenglish
collection-langfrench
collection-langgerman
collection-langgreek
collection-langitalian
collection-langpolish
collection-langportuguese
collection-langspanish
collection-langjapanese
collection-langkorean
collection-langarabic
collection-langchinese
collection-langcjk
collection-langother
collection-xetex
collection-humanities
collection-mathscience
collection-pictures
collection-publishers
collection-music
collection-games
collection-fontsrecommended
collection-latexrecommended
collection-binextra
collection-bibtexextra
collection-formatsextra
collection-latexextra
acro
acronym
acroterm
enumitem
enumitem-zref
yfonts
doublestroke
was
xypic
barcodes
qrcode
lastpage
appendix
changebar
footmisc
multirow
overpic
subfigure
titlesec
siunitx
combelow
csquotes
etoolbox
etextools
idxlayout
bidi
filecontents
eplain
texsis
mltex
lollipop
moreverb
indextools
splitindex
eepic
bigfoot
xstring
showexpl
cweb-latex
hypdvips
ptex
perltex
collection-pstricks
$(collection_by_size fontsextra 20000 || exit 1)
" texmfget base || exit 1
# Put all remaining stuff in "extra"
# Pull some bin-packages from tlnet, which aren't provided by the texlive source tarball
PACKAGES="
collection-texworks
collection-wintools
wasy2-ps
" texmfget extra || exit 1
# Call "fonts" at the end to add remaining fonts
PACKAGES="
collection-fontsextra
" texmfget fonts || exit 1
}
# ==== Nothing to edit beyond this line ====
usage () {
echo
echo "Generate texmf trees/editions based on collections/packages and their dependencies."
echo "./texmf_get.sh [base|docs|extra|fonts]"
echo
echo "-base: texfiles, no docs"
echo "-docs: docs of -base"
echo "-extra: remaining texfiles and docs"
echo "-fonts: fonts and docs"
echo
echo "Only new/updated/missing tex packages are downloaded."
echo "The first run takes \"long\", tex packages(about 2500Mb)"
echo "need to be downloaded and metafiles are generated."
echo "To check out a new version/release, delete"
echo "$db"
echo "A new ascii database is pulled on the next run,"
echo "and a new version yymmdd is set."
echo
echo "All generated tarballs, logs etc. are going to"
echo "$TMP"
echo
}
collection_by_size () {
# from collection $1, pull packages smaller $2 bytes
start_n="$(grep -n ^"name collection-$1"$ $db | cut -d':' -f1)"
# find end of package/collection
for emptyline in $emptylines
do
if [ "$emptyline" -gt "$start_n" ]
then
end_n=$emptyline
break
fi
done
extrapackages="$(sed "${start_n},${end_n}!d" $db | grep ^"depend " | grep -v ^"depend collection" | sed "s/^depend//g" )"
# add if smaller than ...
for checksize in $extrapackages
do
package_meta $checksize || exit 1
size=$(grep ^"containersize " $texmf/$checksize.meta | cut -d' ' -f2)
# for $2, e.g. 3000 means 3kb
[ $size -lt $2 ] && echo $checksize
done
}
package_meta () {
if [ ! -s "$texmf/$1.meta" ]
then
# collection start linenumer
start_n="$(grep -n ^"name ${1}"$ $db | cut -d':' -f1)"
[ -z "$start_n" ] && echo "$1 was not found in $db, bye." && exit 1
# find end of package/collection
for emptyline in $emptylines
do
if [ "$emptyline" -gt "$start_n" ]
then
end_n=$emptyline
break
fi
done
# Don't handle collections as dependency of other collections
sed "${start_n},${end_n}!d;/^depend collection/d" $db > $texmf/$1.meta
fi
}
download () {
# Download packages, if not already available. Not every packages has a corresponding .doc package.
# Try three times if package isn't present, with -t1 to get another mirror the second time
cd $texmf
if [ ! -s "${1}${flavour}.tar.xz" ]
then
for run in {1..10}
do
wget -t1 -c ${mirror}archive/${1}${flavour}.tar.xz
[ -s "${1}${flavour}.tar.xz" ] && break
done
fi
# If no success by downloading, write error log
[ ! -s ${1}${flavour}.tar.xz ] && echo "Downloading ${1}${flavour}.tar.xz did not work, writing to $errorlog" && echo "$VERSION" >> $errorlog && echo "Error downloading ${1}${flavour}.tar.xz" >> $errorlog && exit 1
# check sha512, give three tries for downloading again(diffrent mirrors are used automatically)
if [ "$flavour" = ".doc" ]
then
sha512="$(grep ^doccontainerchecksum $texmf/$1.meta | cut -d' ' -f2 )"
else
sha512="$(grep ^containerchecksum $texmf/$1.meta | cut -d' ' -f2 )"
fi
for run in {1..10}
do
if [ "$(sha512sum ${1}${flavour}.tar.xz | cut -d' ' -f1 )" != "$sha512" ]
then
# Download (hopefully) newer file
rm ${1}${flavour}.tar.xz
wget -t1 -c ${mirror}archive/${1}${flavour}.tar.xz
else
break
fi
done
# check sha512 again, exit if it fails
if [ "$(sha512sum ${1}${flavour}.tar.xz | cut -d' ' -f1 )" != "$sha512" ]
then
echo "sha512sum $(sha512sum ${1}${flavour}.tar.xz | cut -d' ' -f1 ) of"
echo "${package}${flavour}.tar.xz doesn't match with $db"
# delete metafile on failure to get generated again on next run, where new $db may be in use
rm $texmf/$1.meta
echo "sha512sum $sha512"
echo "Delete ${db}* to be current again, and try again."
exit 1
fi
}
untar () {
# leave if $1 has no content. Therea collections with basically no used package, e.g. texworks
if [ -s "$1" ]
then
while read package
do
# untar all packages, check for relocation, "relocate 1" -> untar in texmf-dist
download $package || exit 1
# untar the tex package
unset relocated
[ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && relocated="-C texmf-dist"
tar vxf ${package}${flavour}.tar.xz --exclude tlpkg $relocated || exit 1
# if binaries are present, put them in texmf-dist
[ -d bin ] && cp -a bin texmf-dist && rm -rf bin
if [ "$flavour" = ".doc" ]
then
size=$(( $(grep ^doccontainersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
else
size=$(( $(grep ^containersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
fi
shortdesc="$(grep ^shortdesc $texmf/$package.meta | cut -d' ' -f2- )"
echo "$size Kb, $package$flavour: $shortdesc" >> $output.meta
#grep ^"execute addMap" $texmf/$package.meta | sed "s/^execute //g" >> $output.updmap.cfg
done < $1
# copy packages index to texmf-dist, so included packages are known in later installation
cat $output.meta >> $output.$edition.meta
# cleanup
[ -f $output.meta ] && rm $output.meta
fi
}
remove_cruft () {
# Remove m$-stuff, ConTeXt single-user-system stuff, source leftovers and pdf-versions of manpages
rm -rf texmf-dist/source
rm -rf texmf-dist/scripts/context/stubs/source/
find texmf-dist/ -type d -name 'win32' -exec rm -rf {} +
find texmf-dist/ -type d -name 'win64' -exec rm -rf {} +
find texmf-dist/ -type d -name 'mswin' -exec rm -rf {} +
find texmf-dist/ -type d -name 'win' -exec rm -rf {} +
find texmf-dist/ -type d -name 'setup' -exec rm -rf {} +
find texmf-dist/ -type d -name 'install' -exec rm -rf {} +
find texmf-dist/ -type f -name '*.bat' -delete
find texmf-dist/ -type f -name '*.bat.w95' -delete
find texmf-dist/ -type f -name '*win32*' -delete
find texmf-dist/ -type f -name 'winansi*' -delete
find texmf-dist/ -type f -name '*-man.pdf' -delete
# Remove zero-length files, as these appear e.g. in hyph-utf8 tex-package.
find . -type f -size 0c -delete
}
texmfget () {
# make sure no package is added more than once.
echo "Preparing index of packages to be added to -${1} ..."
echo "$PACKAGES" | sed "s/[[:space:]]//g;/^$/d" > $collections_tobedone
# Remove outputfile if already present
>$output
>$output_doc
# Only do something if $collection wasn't already done before
while [ -s $collections_tobedone ]
do
collection=$(tail -n1 $collections_tobedone)
# continue with next collection if collection was already done
if [ -s "$collections_done" ]
then
grep -w "^${collection}$" $collections_done &> /dev/null
if [ $? = 0 ]
then
# remove from $collections_tobedone
sed -i "/^$collection$/d" $collections_tobedone
if [ -n "$(grep "^${collection} added to" $logfile)" ]
then
echo "$collection already added " >> $logfile
fi
continue
fi
fi
package_meta $collection || exit 1
# Don't handle collections as dependency of other collections, as this destroys control over what packages to be added
# If $collection is a singel package(not a collection-), add it here
if [ -n "$(head -n1 $texmf/$collection.meta | fgrep -v "name collection" )" ]
then
# if package contains only docs, add to docpackages
if [ -z "$(grep ^runfiles $texmf/$collection.meta)" -a -n "$(grep ^docfiles $texmf/$collection.meta)" ]
then
sed -i "/^$collection$/d" $collections_tobedone
echo "$collection" >> $collections_done
echo "$collection" >> $output_doc
echo "$collection added to -docs $1" >> $logfile
continue
fi
# if package contains also docs, add also to docpackages
if [ -n "$(grep ^docfiles $texmf/$collection.meta)" ]
then
echo "$collection" >> $output_doc
echo "$collection added to -docs $1" >> $logfile
fi
echo "$collection" >> $output
echo "$collection added to -$1" >> $logfile
fi
# add dependend packages, but no binary(ARCH) and no packages conataining a '.'. Packges with dot indicate binary/texlive-manager/windows packages
grep ^"depend " $texmf/$collection.meta | grep -v "ARCH$" | grep -v '\.' | cut -d' ' -f2- > $dependencies
if [ -s "$dependencies" ]
then
echo "----------------" >> $logfile
echo "Dependencies of $collection: $(cat $dependencies | tr '\n' ' ')" >> $logfile
for dependency in $(cat $dependencies)
do
if [ -n "$(grep ^"${dependency}"$ $collections_done)" ]
then
sed -i "/^${dependency}$/d" $dependencies
continue
else
for exclude in $global_exclude
do
if [ "$exclude" = "$dependency" ]
then
sed -i "/^${exclude}$/d" $dependencies
echo "$exclude excluded, see \$global_exclude" >> $logfile
fi
done
fi
done
cat $dependencies >> $collections_tobedone
echo "----------------" >> $logfile
fi
sed -i "/^${collection}$/d" $collections_tobedone
echo "$collection" >> $collections_done
done
# handle package index list per edition
cat $output >> $TMP/packages.$1
# handle doc package index, one for each edition
cat $output_doc >> $TMP/packages.$1.doc
# untar only one $edition, untar docs together with -extra/-fonts edition
if [ "$1" = $edition -o docs = $edition ]
then
cd $texmf
# Cleanup tar-directory
[ -d $texmf/texmf-dist ] && rm -rf $texmf/texmf-dist
mkdir $texmf/texmf-dist
# Make tarball/checksum reproducible by setting mtime(clamp-mtime), owner, group and sort content
# --clamp-mtime --mtime doesn't work with tar 1.13, when makepkg creates the tarball:
# tar-1.13: time_t value 9223372036854775808 too large (max=68719476735)
case $edition in
base)
unset flavour
untar $output || exit 1
remove_cruft || exit 1
tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
rm -rf texmf-dist
;;
extra|fonts)
unset flavour
untar $output || exit 1
export flavour=".doc"
untar $output_doc || exit 1
remove_cruft || exit 1
#tar vrf $tarball --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
rm -rf texmf-dist
;;
docs)
export flavour=".doc"
# only add -base docs to -docs
if [ $1 = base ]
then
untar $output_doc || exit 1
remove_cruft || exit 1
#tar vrf $tarball --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
rm -rf texmf-dist
fi
;;
esac
fi
}
# Main
# release mirror
mirror="http://mirror.ctan.org/systems/texlive/tlnet/"
# pre-test mirror 2016
# mirror="http://ftp.cstug.cz/pub/tex/local/tlpretest/"
LANG=C
TMP=$PWD/tmp
output=$TMP/packages
output_doc=$TMP/packages.doc.tmp
errorlog=$TMP/error.log
texmf=$TMP/texmf
db=$TMP/texlive.tlpdb
tmpfile=$TMP/tmpfile
collections_done=$TMP/done
collections_tobedone=$TMP/tobedone
corepackages=$TMP/corepackages
allcollections=$TMP/allcollections
metafonts=$TMP/metafonts
manpages=$TMP/manpages
dependencies=$TMP/deps
packages_base=$TMP/packages.base
packages_extra=$TMP/packages.extra
packages_fonts=$TMP/packages.fonts
packages_metafont=$TMP/packages.metafont
packages_manpages=$TMP/packages.manpages
case "$1" in
base|docs|extra|fonts) edition=$1; echo "Building $edition tarball ...";;
*) usage; exit 0 ;;
esac
mkdir -p $texmf
cd $TMP
# Set VERSION, get texlive.tlpdb and keep unshorten $db.orig
if [ ! -s ${db}.orig -o ! -s $db ]
then
echo $MAJORVERSION.$(date +%y%m%d) > VERSION
wget -c -O ${db}.orig ${mirror}tlpkg/texlive.tlpdb
# remove most content from $db to be faster on later processing.
# keep dependencies/manpages/metafonts/binfiles/shortdesc/sizes
egrep '^\S|^ RELOC/doc/man|^ texmf-dist/doc/man/man|^ RELOC/doc/info/|^ texmf-dist/doc/info/|^ texmf-dist/fonts/source/public/|^ RELOC/fonts/source/public|^ bin|^$' ${db}.orig | grep -v ^longdesc > $db
# As $db (might be)/is new, remove the meta-files, might created again with (pontentionally) new content
rm -rf $texmf/*.meta
fi
# Get linenumbers of empty lines from $db
emptylines="$(grep -n ^$ $db | cut -d':' -f1)"
# Provide TLCore packages for -base, as these packages(and their dependencies) should be present in any case.
grep -B1 ^'category TLCore' $db | grep -v ^'category TLCore' | grep -v ^-- | grep -v '\.' | cut -d' ' -f2 > $corepackages
# Make a list of all collections
grep ^"name collection-" $db | cut -d' ' -f2 > $allcollections
# globally exclude from $corepackages
for exclude in $global_exclude
do
sed -i "/^${exclude}$/d" $corepackages
done
VERSION=$(cat $TMP/VERSION)
tarball=$TMP/texlive-$edition-$VERSION.tar
# set logfile
logfile=$TMP/$VERSION.log
# reset some files
>$logfile
>$tarball
>$collections_done
>$metafonts
>$manpages
>$packages_metafont
>$packages_manpages
>$packages_base
>$packages_extra
>$packages_fonts
>$packages_base.doc
>$packages_extra.doc
>$packages_fonts.doc
>$TMP/packages.$edition.meta
# put the editions base/extra/fonts together
texmf_editions || exit 1
# Check if all collections ar part in at least one edition
while read collection
do
grep -w "$collection" $collections_done &> /dev/null
if [ $? != 0 ]
then
echo "Error: $collection was not handled, edit packages/collections inthe texmfget function in $0." | tee -a $logfile
exit 1
fi
done < $allcollections
# meta data about added packages
sort -n $output.$edition.meta > $tmpfile
mv $tmpfile $output.$edition.meta
# cleanup
rm $allcollections
rm $corepackages
rm $collections_done
rm $collections_tobedone
rm $output
rm $output_doc
rm $dependencies
[ ! -d texmf-dist ] && mkdir texmf-dist
# include all metafonts in base package, plus packages which misses font-mf tag on CTAN
echo "Looking for metafont files to be included in -base ..."
for metafont in $(paste -s $packages_extra $packages_fonts )
do
if [ -n "$(egrep "(fonts/source/public/)" $texmf/$metafont.meta )" ]
then
# include all metafonts in -base, write index for later exclution from other editions.
package_meta $metafont || exit 1
echo "Adding metafonts from $metafont to -base"
unset flavour
download $metafont || exit 1
unset relocated
pathprefix="texmf-dist/"
[ -n "$(grep -w ^"relocated 1" $texmf/$metafont.meta)" ] && relocated="-C texmf-dist" && unset pathprefix
tar vxf $texmf/$metafont.tar.xz $relocated ${pathprefix}fonts/source ${pathprefix}tex/latex 2>/dev/null | sed "s/^fonts/texmf-dist\/fonts/g;s/^tex\//texmf-dist\/tex\//g" >> $metafonts
echo $metafont >> $packages_metafont
fi
done
# include manpages/GNU infofiles in -base, write index for later exclution from other editions.
# In -extra there should not be any manpage left.
echo "Looking for manpages/GNU infofiles to be included in -base ..."
for package in $(paste -s $packages_metafont $packages_base.doc | sort -u)
do
if [ -n "$(egrep "(doc/man/man|doc/info/)" $texmf/$package.meta )" ]
then
echo "Adding manpage from $package to -base"
flavour=".doc" download $package || exit 1
unset relocated
pathprefix="texmf-dist/"
[ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && relocated="-C texmf-dist" && unset pathprefix
tar vxf $texmf/${package}.doc.tar.xz --exclude "*.man[15].pdf" $relocated ${pathprefix}doc/man/ ${pathprefix}doc/info 2>/dev/null | sed "s/^doc/texmf-dist\/doc/g" >> $manpages
echo "$package" >> $packages_manpages
fi
done
case $edition in
base)
# Content info
cat << EOF | gzip -9 >> $texmf/texmf-dist/packages.$edition.gz
Content of -$edition:
$(sort $packages_base)
Metafonts from packages:
$(sort $packages_metafont)
Manpages from packages:
$(sort $packages_manpages)
EOF
# add manpages/metafonts to the tarball
tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
# cleanup extracted metafonts/manpages
rm -rf texmf-dist
# handle koma-script docs, the author wants the docs to be shipped along, html doc seems sufficient
tar f $tarball --delete $(tar tf $tarball | grep /doc/.*koma-script.*pdf)
;;
extra|fonts)
# cleanup extracted metafonts/manpages
rm -rf texmf-dist
echo "Removing manpages/metafonts from -extra/-fonts/-docs which now reside in -base"
tar -f $tarball --delete $(paste $manpages $metafonts) 2>/dev/null
# content info
mkdir texmf-dist
echo "Content of -$edition, including documentation:" > $texmf/texmf-dist/packages.$edition
sort $TMP/packages.$edition >> $texmf/texmf-dist/packages.$edition
gzip -9 $texmf/texmf-dist/packages.$edition
tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
;;
docs)
# cleanup extracted metafonts/manpages
rm -rf texmf-dist
echo "Removing manpages/metafonts from -extra/-fonts/-docs which no reside in -base"
tar -f $tarball --delete $(paste $manpages $metafonts) 2>/dev/null
# content info, this edition conains all docs from -base
mkdir texmf-dist
echo "Content of -$edition, documentation for -base:" > $texmf/texmf-dist/packages.$edition
sort $packages_base.doc >> $texmf/texmf-dist/packages.$edition
gzip -9 $texmf/texmf-dist/packages.$edition
tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
;;
esac
# compress the tarball as everything is in place now
echo "Compressing $tarball ..."
[ -f $tarball.xz ] && rm $tarball.xz
xz -9 -T0 $tarball || exit 1
md5sum $tarball.xz
ls -lh $tarball.xz
echo "Logfile: $logfile"
