summaryrefslogtreecommitdiffstats
path: root/source/t/texlive/prep/texmf_get.sh
diff options
context:
space:
mode:
Diffstat (limited to 'source/t/texlive/prep/texmf_get.sh')
-rwxr-xr-x[-rw-r--r--]source/t/texlive/prep/texmf_get.sh856
1 files changed, 579 insertions, 277 deletions
diff --git a/source/t/texlive/prep/texmf_get.sh b/source/t/texlive/prep/texmf_get.sh
index 396fdcebf..fab74ce5f 100644..100755
--- a/source/t/texlive/prep/texmf_get.sh
+++ b/source/t/texlive/prep/texmf_get.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# texmf_get.sh (c) 2016-2017 Johannes Schoepfer, Germany, slackbuilds[at]schoepfer[dot]info
+# texmf_get.sh (c) 2016-2018 Johannes Schoepfer, Germany, slackbuilds[at]schoepfer[dot]info
# All rights reserved.
#
# Redistribution and use of this script, with or without modification, is
@@ -20,16 +20,15 @@
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
-# V 0.14.2
+# V 15.0.0
#
# Prepare xz-compressed tarballs of texlive-texmf-trees based on texlive.tlpdb
-# This script takes care of dependencies(as far as these are present in texlive.tlpdb) of collections and packages,
-# and that every texlive-package is included only once.
-# The editions(base/extra/fonts/docs) should contain nobinaries(exception biber)
-# -base: the most usefull stuff, all metafonts, all manpages, the most
-# binaries/scripts, 65mb 2017-11-07
+# This script takes care of dependencies(as far as these are present in texlive.tlpdb)
+# of collections and packages, and that every texlive-package is included only once.
+# The editions(base/extra/docs) should contain no binaries(exception biber)
+# -base: the most usefull stuff, most binaries/scripts,
+# manpages for compiled binaries 65mb 2017-11-07
# -docs: -base documentation only, no manpages/GNU infofiles
-# -fonts: fonts only, no metafonts
# -extra: remaining stuff
#
# texlive netarchive policy: Every package is included as dependency
@@ -39,231 +38,244 @@
# package source: http://mirror.ctan.org/systems/texlive/tlnet/archive/
#set -e
-MAJORVERSION=2017
+MAJORVERSION=2018
-# globally excluded packages, which e.g. are
+# -excluded packages, which are
# -useless without tlmgr-installer
# -non-linux
-# -covered by an external package
-# -obsolete
+# -covered by an external package e.g. asymptote on SBo
+# -obsolete
+# -binaries provided already by texlive.Slackbuild
+# -does only contain sources, or hyphen directives, e.g. metatype1, patch, ...
+# -does only contain hyphen directives, e.g. hyphen-farsi ...
global_exclude="
+ hyphen-farsi
+ hyphen-arabic
asymptote
tlcockpit
tlshell
+ texlive.infra
+ texlive-docindex
+ texlive-scripts
+ texlive-msg-translations
texosquery
+ texliveonfly
+ texworks
aleph
omega
antomega
omegaware
lambda
otibet
+ ocherokee
+ oinuit
cslatex
+ bibarts
+ metatype1
+ patch
+ wintools.win32
+ dviout.win32
+ "
+
+ # special packages, move their type1 fonts(if metfonts are present)
+ # and/or docs to -extra
+special_packages="
+ koma-script
+ velthuis
+ armtex
+ montex
+ vntex
+ pl
+ cc-pl
+ cs
+ musixtex-fonts
+ tipa
+ cbfonts
+ ptex-fonts
+ uptex-fonts
+ "
+#Todo: split type1 fonts, or keep subset of type1 fonts in base
+# cm-super
+
+# keep precompiled binaries, list binary, not package name
+keep_precompiled="
+ biber
"
texmf_editions () {
- # At first, $corepackages for the base, and other essentials
+
+ # "excludes from -base", also dependencies are excluded
+ PACKAGES="
+ cm-super
+ biber
+ bib2gls
+ knitting
+ pgfornament
+ pgfplots
+ pst-cox
+ pst-poker
+ pst-vectorian
+ pst-geo
+ bclogo
+ $(grep ^"name .*biblatex" $db | cut -d' ' -f2 )
+ " texmfget extra || exit 1
+
+ # packages/collections and their dependencies for -base
PACKAGES="
$(cat $corepackages)
+ etoolbox
+ xcolor
+ memoir
+ velthuis
+ wasy
+ ptex
+ platex
+ revtex
+ uptex
+ uplatex
collection-basic
collection-latex
collection-metapost
collection-plaingeneric
- collection-fontutils
collection-luatex
collection-context
- cbfonts-fd
- xetex-devanagari
- " texmfget base || exit 1
+ collection-fontutils
+ collection-langczechslovak
+ collection-langeuropean
+ collection-langenglish
+ collection-langfrench
+ collection-langitalian
+ collection-langpolish
+ collection-langportuguese
+ collection-langspanish
+ collection-latexrecommended
+ $(collection_by_size fontsextra 70000 || exit 1)
+ $(collection_by_size publishers 10000 || exit 1)
+ " texmfget base
- # Now the fonts package to make sure that big fonts like cm-super are not included elsewhere as dependency
- PACKAGES="
- cm-super
- cbfonts
- ipaex
- wadalab
- ethiop-t1
- fonts-tlwg
- uhc
- fandol
- arphic
- arphic-ttf
- nanumtype1
- baekmuk
- unfonts-extra
- unfonts-core
- " texmfget fonts || exit 1
-
- # put some stuff in "extra" to before these make their way into "base" as dependency
+ # packages/collections and their dependencies for -extra
PACKAGES="
- $(grep ^"name biblatex" $db | grep -v '\.' | cut -d' ' -f2 )
- biber.x86_64-linux
- biber.i386-linux
- bib2gls
- bibarts
- arara
- latex2nemeth
- ghsystem
- adobemapping
- knitting
- pgfornament
- pgfplots
+ amiri
arabi
- nwejm
- uantwerpendocs
- sduthesis
- stellenbosch
+ arabi-add
+ arara
+ arev
+ bangorcsthesis
+ beamer2thesis
+ beamertheme-detlevcm
+ beamertheme-epyt
+ beamertheme-saintpetersburg
+ beebe
+ bhcexam
+ bxtexlogo
+ churchslavonic
+ collection-fontsextra
+ collection-langchinese
+ collection-langcjk
+ collection-langjapanese
+ collection-langkorean
+ collection-publishers
+ collection-texworks
+ collection-wintools
+ ctan-o-mat
+ ctanify
+ ctanupload
+ dad
+ ethiop-t1
+ fibeamer
fithesis
+ ghsystem
gregoriotex
+ hustthesis
+ ijsra
+ iwhdp
+ jpsj
+ kerkis
+ ketcindy
+ kpfonts
+ langsci
+ latex2nemeth
+ libertine
lilyglyphs
- musixtex-fonts
- beebe
- velthuis
+ lni
+ luatexko
+ media9
+ musuos
mwe
+ newtx
+ nwejm
+ padauk
+ pdfwin
pdfx
- media9
- pst-cox
- pst-poker
- pst-vectorian
- pst-geo
+ powerdot-tuliplab
+ powerdot-FUBerlin
quran
- ijsra
- fibeamer
- udesoftec
- xduthesis
- hustthesis
- bangorcsthesis
- sapthesis
- uowthesis
- cs
- pl
- cc-pl
- tipa
- kerkis
- amiri
- cns
- vntex
- montex
- xcharter
- fonts-churchslavonic
- japanese-otf
+ resumecls
sanskrit-t1
+ sapthesis
+ sduthesis
+ seuthesix
+ simurgh
skaknew
- padauk
+ stellenbosch
+ suanpan
+ tudscr
+ uantwerpendocs
+ ucs
+ udesoftec
+ universalis
+ uowthesis
+ wasy2-ps
+ xduthesis
+ xetexko
+ xq
" texmfget extra || exit 1
-
- # Completing the base
+
+ # What's left, for base again
PACKAGES="
+ collection-fontsrecommended
+ collection-xetex
collection-langcyrillic
- collection-langczechslovak
- collection-langeuropean
- collection-langenglish
- collection-langfrench
+ collection-langarabic
collection-langgerman
collection-langgreek
- collection-langitalian
- collection-langpolish
- collection-langportuguese
- collection-langspanish
- collection-langjapanese
- collection-langkorean
- collection-langarabic
- collection-langchinese
- collection-langcjk
collection-langother
- collection-xetex
collection-humanities
collection-mathscience
collection-pictures
- collection-publishers
+ collection-pstricks
collection-music
collection-games
- collection-fontsrecommended
- collection-latexrecommended
collection-binextra
collection-bibtexextra
collection-formatsextra
collection-latexextra
- acro
- acronym
- acroterm
- enumitem
- enumitem-zref
- yfonts
- doublestroke
- was
- xypic
- barcodes
- qrcode
- lastpage
- appendix
- changebar
- footmisc
- multirow
- overpic
- subfigure
- titlesec
- siunitx
- combelow
- csquotes
- etoolbox
- etextools
- idxlayout
- bidi
- filecontents
- eplain
- texsis
- mltex
- lollipop
- moreverb
- indextools
- splitindex
- eepic
- bigfoot
- xstring
- showexpl
- cweb-latex
- hypdvips
- ptex
- perltex
- collection-pstricks
- $(collection_by_size fontsextra 20000 || exit 1)
" texmfget base || exit 1
- # Put all remaining stuff in "extra"
- # Pull some bin-packages from tlnet, which aren't provided by the texlive source tarball
- PACKAGES="
- collection-texworks
- collection-wintools
- wasy2-ps
- " texmfget extra || exit 1
-
- # Call "fonts" at the end to add remaining fonts
- PACKAGES="
- collection-fontsextra
- " texmfget fonts || exit 1
-
}
# ==== Nothing to edit beyond this line ====
usage () {
echo
- echo "Generate texmf trees/editions based on collections/packages and their dependencies."
- echo "./texmf_get.sh [base|docs|extra|fonts]"
+ echo "Generate texmf trees/editions based on collections/packages"
+ echo "and their (hard)dependencies."
+ echo "./texmf_get.sh [base|docs|extra|lint]"
echo
echo "-base: texfiles, no docs"
echo "-docs: docs of -base"
echo "-extra: remaining texfiles and docs"
- echo "-fonts: fonts and docs"
+ echo "[lint]: compare filename contents of all generated editions,"
+ echo " to detect overlapping files"
echo
echo "Only new/updated/missing tex packages are downloaded."
echo "The first run takes \"long\", tex packages(about 2500Mb)"
- echo "need to be downloaded and metafiles are generated."
+ echo "need to be downloaded."
echo "To check out a new version/release, delete"
echo "$db"
- echo "A new ascii database is pulled on the next run,"
- echo "and a new version yymmdd is set."
+ echo "A new ascii index file/database(texlive.tlpdb) is will be"
+ echo "pulled on the next run, and a new version yymmdd will be set."
echo
echo "All generated tarballs, logs etc. are going to"
echo "$TMP"
@@ -362,36 +374,116 @@ download () {
}
untar () {
- # leave if $1 has no content. Therea collections with basically no used package, e.g. texworks
+ # leave if $1 has no content
if [ -s "$1" ]
then
while read package
do
+ echo "untar $package"
# untar all packages, check for relocation, "relocate 1" -> untar in texmf-dist
download $package || exit 1
- # untar the tex package
- unset relocated
- [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && relocated="-C texmf-dist"
- tar vxf ${package}${flavour}.tar.xz --exclude tlpkg $relocated || exit 1
+ # untar the tex package, relocate to texmf-dist if necessary, binpackages always need relocation
+ relocated='.'
+ [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" -o -n "$(grep ^"binfiles " $texmf/$package.meta)" ] && relocated="texmf-dist"
+ # if not .doc package, investigate files for dependencies/provides
+ if [ -n "$flavour" ]
+ then
+ tar xf ${package}${flavour}.tar.xz --exclude tlpkg -C $relocated || exit 1
+ else
+ #tar vxf ${package}${flavour}.tar.xz --exclude tlpkg -C $relocated
+ tar vxf ${package}${flavour}.tar.xz --exclude tlpkg -C $relocated | egrep '\.sty$|\.bbx$|\.cls$' > $texmf/$package.deps
+ if [ -n "$texmf/$package.deps" ]
+ then
+ unset provide
+ unset depends
+ for depfile in $(cat $texmf/$package.deps)
+ do
+ filename="$( echo $depfile | rev | cut -d'.' -f2- | cut -d'/' -f1 | rev)"
+ # always add $filename as "ProvidesPackage", if it's a .sty
+ echo $depfile | grep '\.sty'$ &>/dev/null
+ [ $? = 0 ] && provide+="${filename},"
+ # remove comments, if there are backslashes ignore that content, except it is \filename
+ #provide+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\\\ProvidesPackage{\([^}]*\)}.*/\1/p" | sed "s/\\\filename/$filename/g;/\\\/d" | sort -u | tr '\n' ',')"
+ #sed -z "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\)*.\n//g" | sed "s/[[:space:]]//
+ #provide+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -z "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\)*.\n//g" | sed "s/[[:space:]]// | sed -n "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\){\([^}]*\)}.*/\2/p" | sed "s/\\\filename/$filename/g;s/\\\ExplFileName/$filename/g;/\\\/d" | sed "s/\(\.sty$\|\.cls$\)//g" | sort -u | tr '\n' ',')"
+ provide+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -z "s/\(Package\|ExplPackage\|File\|Class\)\n/\1/g" | sed "s/[[:space:]]//" | sed -n "s/.*\\\Provides\(Package\|ExplPackage\|File\|Class\){\([^}]*\)}.*/\2/p" | sed "s/\\\filename/$filename/g;s/\\\ExplFileName/$filename/g" | sed "s/\(\.sty$\|\.cls$\)//g" | sort -u | tr '\n' ',')"
+ #depends+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\(\\\require\|\\\use\)package{\([^}]*\)}.*/\2/p" | sed "/\\\/d" | sort -u | tr '\n' ',')"
+ depends+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\(\\\require\|\\\use\)package{\([^}]*\)}.*/\2/p" | sort -u | tr '\n' ',')"
+ #depends+="$(sed "s/%.*//g" $texmf/$relocated/$depfile | sed -n "s/.*\\\\(require\|use\)package{\([^}]*\)}.*/\2/p" | sed "/\\\/d" | sort -u | tr '\n' ',')"
+ done
+ if [ -n "$provide" ]
+ then
+ #echo "$package $provide | sort -u | tr '\n' ',' " >> $TMP/provides.run.$edition
+ echo "$package $provide" >> $TMP/provides.run.$edition
+ fi
+ if [ -n "$depends" ]
+ then
+ echo "$package $depends" >> $TMP/depends.run.$edition
+ fi
+ fi
+ fi
+
+ # Check for binaries, delete them as these should be provided
+ # by texlive.Slackbuild, keep symlinks and scripts
- # if binaries are present, put them in texmf-dist
- [ -d bin ] && cp -a bin texmf-dist && rm -rf bin
+ unset binaries
+ unset scripts
+ for arch in $platforms
+ do
+ if [ -d $texmf/texmf-dist/bin/$arch ]
+ then
+ [ ! -d $texmf/texmf-dist/linked_scripts ] \
+ && mkdir $texmf/texmf-dist/linked_scripts
+ # rewrite link target to fit systemwide installation
+ for link in $(find $texmf/texmf-dist/bin/$arch -type l)
+ do
+ ln -sf $(readlink $link | sed "s/^..\/..\(.*\)/..\/share\1/" ) $link || exit 1
+ done
+ # move symlinks/scripts to linked_scripts
+ find $texmf/texmf-dist/bin/$arch -type l -exec mv '{}' $texmf/texmf-dist/linked_scripts/ \;
+
+ binaries="$(find $texmf/texmf-dist/bin/$arch -type f -exec file '{}' + | grep -e "executable" -e "shared object" | grep ELF | cut -f 1 -d : )"
+ for bin in $binaries
+ do
+ # keep binaries of special packages
+ for binary in $keep_precompiled
+ do
+ if [ "$(echo $bin | rev | cut -d'/' -f1 | rev )" != "$binary" ]
+ then
+ rm $bin
+ echo $bin | rev | cut -d'/' -f1 | rev >> $binary_removed.$edition
+ fi
+ done
+ done
+ # move scripts to linked-scripts
+ scripts="$(find $texmf/texmf-dist/bin/$arch -type f -exec file '{}' + | grep -wv ELF | cut -f 1 -d : )"
+ for script in $scripts
+ do
+ mv $script $texmf/texmf-dist/linked_scripts/
+ done
+ fi
+ done
+
if [ "$flavour" = ".doc" ]
then
- size=$(( $(grep ^doccontainersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
+ size=$(grep ^doccontainersize $texmf/$package.meta | cut -d' ' -f2)
else
- size=$(( $(grep ^containersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
+ size=$(grep ^containersize $texmf/$package.meta | cut -d' ' -f2)
fi
shortdesc="$(grep ^shortdesc $texmf/$package.meta | cut -d' ' -f2- )"
- echo "$size Kb, $package$flavour: $shortdesc" >> $output.meta
- #grep ^"execute addMap" $texmf/$package.meta | sed "s/^execute //g" >> $output.updmap.cfg
+ echo "$size byte, $package$flavour: $shortdesc" >> $output.meta
+ # make index of uncompressed size of each package
+ echo "$(xz -l --verbose ${package}${flavour}.tar.xz | grep "Uncompressed size" | cut -d'(' -f2 | cut -d' ' -f1 ) byte, $package$flavour: $shortdesc" >> $output.meta.uncompressed
done < $1
# copy packages index to texmf-dist, so included packages are known in later installation
- cat $output.meta >> $output.$edition.meta
+ # don't list binary packages, as the binaries itself are not contained, only the symlinks.
+ cat $output.meta | grep -v '\-linux:' >> $output.$edition.meta
+ cat $output.meta.uncompressed | grep -v '\-linux:' >> $output.$edition.meta.uncompressed
# cleanup
[ -f $output.meta ] && rm $output.meta
+ [ -f $output.meta.uncompressed ] && rm $output.meta.uncompressed
fi
}
@@ -399,19 +491,24 @@ remove_cruft () {
# Remove m$-stuff, ConTeXt single-user-system stuff, source leftovers and pdf-versions of manpages
rm -rf texmf-dist/source
rm -rf texmf-dist/scripts/context/stubs/source/
- find texmf-dist/ -type d -name 'win32' -exec rm -rf {} +
- find texmf-dist/ -type d -name 'win64' -exec rm -rf {} +
- find texmf-dist/ -type d -name 'mswin' -exec rm -rf {} +
- find texmf-dist/ -type d -name 'win' -exec rm -rf {} +
- find texmf-dist/ -type d -name 'setup' -exec rm -rf {} +
- find texmf-dist/ -type d -name 'install' -exec rm -rf {} +
- find texmf-dist/ -type f -name '*.bat' -delete
- find texmf-dist/ -type f -name '*.bat.w95' -delete
- find texmf-dist/ -type f -name '*win32*' -delete
- find texmf-dist/ -type f -name 'winansi*' -delete
- find texmf-dist/ -type f -name '*-man.pdf' -delete
+ find texmf-dist/ -type d -name 'win32' -exec rm -rf {} +
+ find texmf-dist/ -type d -name 'win64' -exec rm -rf {} +
+ find texmf-dist/ -type d -name 'mswin' -exec rm -rf {} +
+ find texmf-dist/ -type d -name 'win' -exec rm -rf {} +
+ find texmf-dist/ -type d -name 'setup' -exec rm -rf {} +
+ find texmf-dist/ -type d -name 'install' -exec rm -rf {} +
+ find texmf-dist/ -type f -name 'uninstall*.sh' -delete
+ find texmf-dist/ -type f -name '*.bat' -delete
+ find texmf-dist/ -type f -name '*.bat.w95' -delete
+ find texmf-dist/ -type f -name '*win32*' -delete
+ find texmf-dist/ -type f -name 'winansi*' -delete
+ find texmf-dist/ -type f -name '*man1.pdf' -delete
+ find texmf-dist/ -type f -name '*man5.pdf' -delete
# Remove zero-length files, as these appear e.g. in hyph-utf8 tex-package.
- find . -type f -size 0c -delete
+ # find texmf-dist/ -type f -size 0c -delete
+ find texmf-dist/ -type f -empty -delete
+ # Remove empty directories recursively
+ find texmf-dist/ -type d -empty -delete
}
texmfget () {
@@ -444,31 +541,64 @@ texmfget () {
fi
package_meta $collection || exit 1
-
- # Don't handle collections as dependency of other collections, as this destroys control over what packages to be added
+
# If $collection is a singel package(not a collection-), add it here
if [ -n "$(head -n1 $texmf/$collection.meta | fgrep -v "name collection" )" ]
then
- # if package contains only docs, add to docpackages
- if [ -z "$(grep ^runfiles $texmf/$collection.meta)" -a -n "$(grep ^docfiles $texmf/$collection.meta)" ]
+ addpackage=no
+ # if package contains docs, add to docpackages
+ if [ -n "$(grep ^docfiles $texmf/$collection.meta)" ]
then
- sed -i "/^$collection$/d" $collections_tobedone
- echo "$collection" >> $collections_done
echo "$collection" >> $output_doc
- echo "$collection added to -docs $1" >> $logfile
- continue
+ echo "$collection added to docs $1" >> $logfile
+ addpackage=yes
fi
- # if package contains also docs, add also to docpackages
- if [ -n "$(grep ^docfiles $texmf/$collection.meta)" ]
+ if [ -n "$(grep ^runfiles $texmf/$collection.meta)" -o -n "$(grep ^binfiles $texmf/$collection.meta)" ]
then
- echo "$collection" >> $output_doc
- echo "$collection added to -docs $1" >> $logfile
+ echo "$collection" >> $output
+ echo "$collection added to -$1" >> $logfile
+ addpackage=yes
+ fi
+ # very package should be added to one edition, abort if that didn't work
+ if [ $addpackage = no ]
+ then
+ echo "$collection doesn't contain any docfiles/runfiles/binfiles"
+ echo "Please exclude package/report to upstream mailinglist tex-live@tug.org, bye."
+ exit 1
fi
- echo "$collection" >> $output
- echo "$collection added to -$1" >> $logfile
fi
+
+ # Don't handle collections as dependency of other collections, as this destroys control over what packages to be added
# add dependend packages, but no binary(ARCH) and no packages conataining a '.'. Packges with dot indicate binary/texlive-manager/windows packages
- grep ^"depend " $texmf/$collection.meta | grep -v "ARCH$" | grep -v '\.' | cut -d' ' -f2- > $dependencies
+
+ grep ^"depend " $texmf/$collection.meta | cut -d' ' -f2- > $dependencies
+
+ if [ -s "$dependencies" ]
+ then
+ # check for .ARCH packages which may be binaries, scripts or links
+ # Binaries should all come from the sourcebuild(exception biber)
+ for dependency in $(cat $dependencies)
+ do
+ echo $dependency | grep '\.ARCH'$ &>/dev/null
+ if [ $? = 0 ]
+ then
+ for arch in $platforms
+ do
+ archpackage="$(echo $dependency | sed "s/\.ARCH$/\.$arch/")"
+ grep ^"name $archpackage"$ $db &>/dev/null && echo "$archpackage" >> $dependencies.verified_arch
+ done
+ else
+ echo $dependency >> $dependencies.verified_arch
+ fi
+ done
+ if [ -f $dependencies.verified_arch ]
+ then
+ mv $dependencies.verified_arch $dependencies
+ else
+ rm $dependencies
+ fi
+ fi
+
if [ -s "$dependencies" ]
then
echo "----------------" >> $logfile
@@ -502,7 +632,7 @@ texmfget () {
# handle doc package index, one for each edition
cat $output_doc >> $TMP/packages.$1.doc
- # untar only one $edition, untar docs together with -extra/-fonts edition
+ # untar only one $edition, untar docs together with -extra edition
if [ "$1" = $edition -o docs = $edition ]
then
cd $texmf
@@ -514,22 +644,23 @@ texmfget () {
# Make tarball/checksum reproducible by setting mtime(clamp-mtime), owner, group and sort content
# --clamp-mtime --mtime doesn't work with tar 1.13, when makepkg creates the tarball:
# tar-1.13: time_t value 9223372036854775808 too large (max=68719476735)
+ echo "Adding files to $( echo $tarball | rev | cut -d'/' -f1 | rev ) ..."
case $edition in
base)
unset flavour
untar $output || exit 1
remove_cruft || exit 1
- tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+ tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
rm -rf texmf-dist
;;
- extra|fonts)
+ extra)
unset flavour
untar $output || exit 1
export flavour=".doc"
untar $output_doc || exit 1
remove_cruft || exit 1
#tar vrf $tarball --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
- tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+ tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
rm -rf texmf-dist
;;
docs)
@@ -540,7 +671,7 @@ texmfget () {
untar $output_doc || exit 1
remove_cruft || exit 1
#tar vrf $tarball --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
- tar vrf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+ tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
rm -rf texmf-dist
fi
;;
@@ -548,6 +679,48 @@ texmfget () {
fi
}
+lint () {
+
+echo "Comparing content of all editions, this may take a while ...";
+cd $TMP
+# check if all editions of same VERSION are there, take -base as reference
+lint_version=$( ls texlive-base-*tar.xz | head -n1 | cut -d'.' -f2 || exit 1)
+if [ -s texlive-extra-$MAJORVERSION.$lint_version.tar.xz \
+ -a -s texlive-docs-$MAJORVERSION.$lint_version.tar.xz ]
+then
+ for edition in base extra docs
+ do
+ echo "Extracting index of texlive-${edition}-$MAJORVERSION.$lint_version.tar.xz ..."
+ # don't list directories
+ tar tf texlive-${edition}-$MAJORVERSION.$lint_version.tar.xz | grep -v '/'$ > $TMP/packages.$edition.lint
+ done
+
+ # compare content
+ for edition in base extra docs
+ do
+ >$TMP/packages.$edition.lint.dup
+ case $edition in
+ base)
+ echo "check if files of base are present in another edition"
+ while read line
+ do
+ grep ^"$line"$ $TMP/packages.extra.lint >> $TMP/packages.base.lint.dup
+ grep ^"$line"$ $TMP/packages.docs.lint >> $TMP/packages.base.lint.dup
+ done < $TMP/packages.$edition.lint
+ ;;
+ esac
+ done
+else
+ echo "Not all editions are present to lint them. Create them first by"
+ echo "$0 [base|docs|extra]"
+ echo "bye."
+ exit 1
+fi
+
+exit 0
+
+}
+
# Main
# release mirror
@@ -566,31 +739,36 @@ collections_done=$TMP/done
collections_tobedone=$TMP/tobedone
corepackages=$TMP/corepackages
allcollections=$TMP/allcollections
-metafonts=$TMP/metafonts
+binary_removed=$TMP/binaries.removed
manpages=$TMP/manpages
dependencies=$TMP/deps
packages_base=$TMP/packages.base
packages_extra=$TMP/packages.extra
-packages_fonts=$TMP/packages.fonts
-packages_metafont=$TMP/packages.metafont
packages_manpages=$TMP/packages.manpages
+files_split=$TMP/files.split
+platforms="x86_64-linux i386-linux"
+
+mkdir -p $texmf
+cd $TMP
case "$1" in
- base|docs|extra|fonts) edition=$1; echo "Building $edition tarball ...";;
+ base|docs|extra) edition=$1;;
+ lint) lint ;;
*) usage; exit 0 ;;
esac
-
-mkdir -p $texmf
-cd $TMP
+
+echo "Building $edition tarball ..."
# Set VERSION, get texlive.tlpdb and keep unshorten $db.orig
-if [ ! -s ${db}.orig -o ! -s $db ]
+if [ ! -s ${db}.orig -o ! -s $db -o ! -s VERSION ]
then
echo $MAJORVERSION.$(date +%y%m%d) > VERSION
wget -c -O ${db}.orig ${mirror}tlpkg/texlive.tlpdb
# remove most content from $db to be faster on later processing.
- # keep dependencies/manpages/metafonts/binfiles/shortdesc/sizes
- egrep '^\S|^ RELOC/doc/man|^ texmf-dist/doc/man/man|^ RELOC/doc/info/|^ texmf-dist/doc/info/|^ texmf-dist/fonts/source/public/|^ RELOC/fonts/source/public|^ bin|^$' ${db}.orig | grep -v ^longdesc > $db
+ # keep dependencies/manpages/binfiles/shortdesc/sizes
+ egrep \
+ '^\S|^ RELOC/doc/man|^ texmf-dist/doc/man/man|^ RELOC/doc/info/|^ texmf-dist/doc/info/|^ bin|^$' \
+ ${db}.orig | grep -v ^longdesc > $db
# As $db (might be)/is new, remove the meta-files, might created again with (pontentionally) new content
rm -rf $texmf/*.meta
@@ -604,7 +782,19 @@ grep -B1 ^'category TLCore' $db | grep -v ^'category TLCore' | grep -v ^-- | gr
# Make a list of all collections
grep ^"name collection-" $db | cut -d' ' -f2 > $allcollections
-
+
+# translate .ARCH to platforms in excludes, to make .ARCH packages excludeable by $global_exclude
+for exclude in $global_exclude
+do
+ if [ -n "$(echo $exclude | grep '\.ARCH'$ )" ]
+ then
+ for arch in $platforms
+ do
+ global_exclude+=" $(echo $exclude | sed "s/\.ARCH$/\.$arch/")"
+ done
+ global_exclude=${global_exclude/$exclude/}
+ fi
+done
# globally exclude from $corepackages
for exclude in $global_exclude
do
@@ -620,36 +810,34 @@ logfile=$TMP/$VERSION.log
>$logfile
>$tarball
>$collections_done
->$metafonts
+>$files_split
>$manpages
->$packages_metafont
>$packages_manpages
>$packages_base
>$packages_extra
->$packages_fonts
>$packages_base.doc
>$packages_extra.doc
->$packages_fonts.doc
>$TMP/packages.$edition.meta
+>$TMP/packages.$edition.meta.uncompressed
+>$TMP/provides.run.$edition
+>$TMP/depends.run.$edition
+>$binary_removed.$edition
-# put the editions base/extra/fonts together
+# put the editions base/extra together
texmf_editions || exit 1
-# Check if all collections ar part in at least one edition
+# Check if all collections are part in at least one edition
while read collection
do
grep -w "$collection" $collections_done &> /dev/null
if [ $? != 0 ]
then
- echo "Error: $collection was not handled, edit packages/collections inthe texmfget function in $0." | tee -a $logfile
+ echo "Error: $collection was not handled."
+ echo "Edit packages/collections in the texmfget function." | tee -a $logfile
exit 1
fi
done < $allcollections
-# meta data about added packages
-sort -n $output.$edition.meta > $tmpfile
-mv $tmpfile $output.$edition.meta
-
# cleanup
rm $allcollections
rm $corepackages
@@ -659,89 +847,203 @@ rm $output
rm $output_doc
rm $dependencies
+# untar special- and manpage packages to be splitted/moved to other editions
+# splitting special packages, files index
+echo "Prepare index of to be splitted/moved files from -base"
[ ! -d texmf-dist ] && mkdir texmf-dist
-# include all metafonts in base package, plus packages which misses font-mf tag on CTAN
-echo "Looking for metafont files to be included in -base ..."
-for metafont in $(paste -s $packages_extra $packages_fonts )
+for package in $special_packages
do
- if [ -n "$(egrep "(fonts/source/public/)" $texmf/$metafont.meta )" ]
+ echo "Splitting $package"
+ # special packages have to be in -base, as only here are special
+ # tasks done to reduce size of -base edition
+ if [ -z "$( grep ^"$package"$ $packages_base )" ]
then
- # include all metafonts in -base, write index for later exclution from other editions.
- package_meta $metafont || exit 1
- echo "Adding metafonts from $metafont to -base"
- unset flavour
- download $metafont || exit 1
- unset relocated
- pathprefix="texmf-dist/"
- [ -n "$(grep -w ^"relocated 1" $texmf/$metafont.meta)" ] && relocated="-C texmf-dist" && unset pathprefix
- tar vxf $texmf/$metafont.tar.xz $relocated ${pathprefix}fonts/source ${pathprefix}tex/latex 2>/dev/null | sed "s/^fonts/texmf-dist\/fonts/g;s/^tex\//texmf-dist\/tex\//g" >> $metafonts
- echo $metafont >> $packages_metafont
+ echo "$package was not found to be part of -base"
+ echo "Edit \$special_packages in $0"
+ echo "to contain only packages from -base, bye."
+ exit 1
fi
+ unset relocated
+ pathprefix="texmf-dist/"
+ [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && \
+ relocated="-C texmf-dist" && unset pathprefix
+ # avoid big pdf docs which are also present as html
+ # move (big)type1 fonts to -extra
+ # $files_split lists files to be moved from -base -extra
+ if [ $package = "cm-super" ]
+ then
+ # cm-super minimal for -base, create index of extended cm-super
+ tar tf $texmf/$package.tar.xz | sed \
+ "/1000\.pfb$/d;/^tlpkg/d;/\.sty$/d;/\.enc$/d;/\.GS$/d" \
+ | tee -a $files_split > $files_split.tmp
+ else
+ tar tf $texmf/${package}.tar.xz | sed \
+ -ne "/.*doc\/latex\/.*\.pdf$/p" \
+ -ne "/.*fonts\/map\/.*\.map$/p" \
+ -ne "/.*fonts\/enc\/.*\.enc$/p" \
+ -ne "/.*fonts\/afm\/.*\.\(afm\|afm\.gz\)$/p" \
+ -ne "/.*fonts\/type1\/.*\.pfb$/p" \
+ -ne "/.*fonts\/vf\/.*\.vf$/p" \
+ | tee -a $files_split > $files_split.tmp
+ fi
+
+ if [ $edition = base ]
+ then
+ # Calculate package-minimal size, uncompressed and compressed
+ mkdir -p calculate/texmf-dist
+ tar xf $texmf/$package.tar.xz -C calculate/texmf-dist --exclude-from=$files_split.tmp
+ tar cf calculate/calc.tar.xz -I 'xz -9' calculate/texmf-dist
+ size_minimal=$(du -bc calculate/calc.tar.xz | tail -n1 | sed "s/[[:space:]].*//")
+ size_minimal_uncompressed="$(xz -l --verbose calculate/calc.tar.xz | grep "Uncompressed size" | cut -d'(' -f2 | cut -d' ' -f1 )"
+ sed -i \
+ -e "s/^[0-9]* byte, $package: /$size_minimal byte, $package-minimal: /" \
+ $output.base.meta
+ sed -i \
+ -e "s/^[0-9]* byte, $package: /$size_minimal_uncompressed byte, $package-minimal: /" \
+ $output.base.meta.uncompressed
+ rm -rf calculate
+ fi
+
+ if [ $edition = extra ]
+ then
+ mkdir -p calculate/texmf-dist
+ tar xf $texmf/${package}.tar.xz -C calculate/texmf-dist $(paste $files_split.tmp)
+ tar cf calculate/calc.tar.xz -I 'xz -9' calculate/texmf-dist
+ size_extended=$(du -bc calculate/calc.tar.xz | tail -n1 | sed "s/[[:space:]].*//")
+ size_extended_uncompressed="$(xz -l --verbose calculate/calc.tar.xz | \
+ grep "Uncompressed size" | cut -d'(' -f2 | cut -d' ' -f1 )"
+
+ # put new sizes in package index uncompressed
+ sed -i \
+ -e "s/^[0-9]* byte, $package: /$size_extended byte, $package-extended: /" \
+ $output.extra.meta
+ sed -i \
+ -e "s/^[0-9]* byte, $package: /$size_extended_uncompressed byte, $package-extended: /" \
+ $output.extra.meta.uncompressed
+ rm -rf calculate
+ fi
+
+ # untar to provide files for -extra
+ tar xf $texmf/${package}.tar.xz $relocated $(paste $files_split.tmp)
+ if [ $package = "cm-super" ]
+ then
+ # create cm-super- minimal config/maps with 10pt glyphs only
+ sed "s/cm-super/cm-super-minimal/g" $texmf/texmf-dist/dvips/cm-super/config.cm-super \
+ > $texmf/texmf-dist/dvips/cm-super/config-minimal.cm-super
+ for map in t1 t2a t2b t2c ts1 x2
+ do
+ grep 1000 $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-$map.map \
+ > $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-minimal-$map.map
+ sed -i "/.*1000\.pfb/d" $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-$map.map
+ done
+ fi
+
done
+
+# cleanup
+rm $files_split.tmp
+# fix relocation in index for splitted packages
+sed -i \
+ -e "s|^doc|texmf-dist\/doc|g" \
+ -e "s|^fonts|texmf-dist\/fonts|g" \
+ -e "s|^dvips|texmf-dist\/dvips|g" \
+ $files_split
+
+# sort meta data about added packages
+sort -n $output.$edition.meta > $tmpfile
+mv $tmpfile $output.$edition.meta
+sort -n $output.$edition.meta.uncompressed > $tmpfile
+mv $tmpfile $output.$edition.meta.uncompressed
+
+sort -u $binary_removed.$edition > $tmpfile
+mv $tmpfile $binary_removed.$edition
+
# include manpages/GNU infofiles in -base, write index for later exclution from other editions.
-# In -extra there should not be any manpage left.
+# In -extra/-docs there should not be any manpage left.
echo "Looking for manpages/GNU infofiles to be included in -base ..."
-for package in $(paste -s $packages_metafont $packages_base.doc | sort -u)
+for package in $(paste -s $packages_base.doc | sort -u)
do
if [ -n "$(egrep "(doc/man/man|doc/info/)" $texmf/$package.meta )" ]
then
- echo "Adding manpage from $package to -base"
+ echo "Adding manpage from $package.doc to -base"
flavour=".doc" download $package || exit 1
unset relocated
pathprefix="texmf-dist/"
- [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] && relocated="-C texmf-dist" && unset pathprefix
- tar vxf $texmf/${package}.doc.tar.xz --exclude "*.man[15].pdf" $relocated ${pathprefix}doc/man/ ${pathprefix}doc/info 2>/dev/null | sed "s/^doc/texmf-dist\/doc/g" >> $manpages
+ [ -n "$(grep -w ^"relocated 1" $texmf/$package.meta)" ] \
+ && relocated="-C texmf-dist" && unset pathprefix
+ tar tf $texmf/${package}.doc.tar.xz | sed \
+ -ne "/.*doc\/man\/.*\.1$/p" \
+ -ne "/.*doc\/man\/.*\.5$/p" \
+ -ne "/.*doc\/info\/.*\.info$/p" \
+ | tee -a $manpages > $manpages.tmp
+ # untar to provide files for -/extra/-docs
+ tar xf $texmf/${package}.doc.tar.xz $relocated $(paste $manpages.tmp)
echo "$package" >> $packages_manpages
fi
done
+# cleanup
+rm $manpages.tmp
+sed -i \
+ -e "s/^doc/texmf-dist\/doc/g" \
+ $manpages
case $edition in
base)
# Content info
cat << EOF | gzip -9 >> $texmf/texmf-dist/packages.$edition.gz
Content of -$edition:
-$(sort $packages_base)
-
-Metafonts from packages:
-$(sort $packages_metafont)
-
-Manpages from packages:
-$(sort $packages_manpages)
+$(sed "/-linux$/d" $packages_base | sort)
EOF
-
- # add manpages/metafonts to the tarball
- tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
- # cleanup extracted metafonts/manpages
- rm -rf texmf-dist
- # handle koma-script docs, the author wants the docs to be shipped along, html doc seems sufficient
- tar f $tarball --delete $(tar tf $tarball | grep /doc/.*koma-script.*pdf)
+#Splitted packages, type1 fonts/docs moved to -extra:
+#$(echo $special_packages)
+#EOF
+
+# add manpages/GNU infofiles to the tarball
+ tar rf $tarball --owner=0 --group=0 --sort=name \
+ texmf-dist/doc/man/ texmf-dist/doc/info/ \
+ texmf-dist/packages.$edition.gz \
+ || exit 1
+# # add cm-super minimal maps/config
+# tar rf $tarball --owner=0 --group=0 --sort=name \
+# texmf-dist/dvips/cm-super/config-minimal.cm-super \
+# --wildcards texmf-dist/fonts/map/dvips/cm-super/cm-super-minimal-*.map \
+# || exit 1
+ echo "Removing files -from base, splitted from special packages to be included in -extra"
+ tar f $tarball --delete $(paste $files_split) || exit 1
;;
- extra|fonts)
- # cleanup extracted metafonts/manpages
- rm -rf texmf-dist
- echo "Removing manpages/metafonts from -extra/-fonts/-docs which now reside in -base"
- tar -f $tarball --delete $(paste $manpages $metafonts) 2>/dev/null
+ extra)
+ echo "Removing manpages from $edition which now reside in -base"
+ tar f $tarball --delete $(paste $manpages) 2>/dev/null
# content info
- mkdir texmf-dist
echo "Content of -$edition, including documentation:" > $texmf/texmf-dist/packages.$edition
- sort $TMP/packages.$edition >> $texmf/texmf-dist/packages.$edition
+ sed "/-linux$/d" $TMP/packages.$edition | sort >> $texmf/texmf-dist/packages.$edition
gzip -9 $texmf/texmf-dist/packages.$edition
- tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+# # remove cm-super minimal config, which resides in -base
+# rm \
+# $texmf/texmf-dist/dvips/cm-super/config-minimal.cm-super \
+# $texmf/texmf-dist/fonts/map/dvips/cm-super/cm-super-minimal-*.map
+ tar rf $tarball --owner=0 --group=0 --sort=name \
+ --exclude texmf-dist/doc \
+ texmf-dist \
+ || exit 1
;;
docs)
- # cleanup extracted metafonts/manpages
- rm -rf texmf-dist
- echo "Removing manpages/metafonts from -extra/-fonts/-docs which no reside in -base"
- tar -f $tarball --delete $(paste $manpages $metafonts) 2>/dev/null
- # content info, this edition conains all docs from -base
- mkdir texmf-dist
+ # add docs splittet from base from special packages, add packages index
+ # content info, this edition contains all docs from -base
echo "Content of -$edition, documentation for -base:" > $texmf/texmf-dist/packages.$edition
sort $packages_base.doc >> $texmf/texmf-dist/packages.$edition
gzip -9 $texmf/texmf-dist/packages.$edition
- tar rf $tarball --owner=0 --group=0 --sort=name texmf-dist || exit 1
+ tar rf $tarball --owner=0 --group=0 --sort=name \
+ texmf-dist/doc/ \
+ texmf-dist/packages.$edition.gz \
+ || exit 1
+ echo "Removing manpages from $edition which now reside in -base"
+ tar f $tarball --delete $(paste $manpages) || exit 1
;;
esac
+
+rm -rf texmf-dist
# compress the tarball as everything is in place now
echo "Compressing $tarball ..."
generated by cgit v1.2.3 (git 2.26.2) at 2024-04-20 15:50:14 +0000