[pacman-dev] [PATCH 1/3] bacman: allow for parallel packaging
Dave Reisner
d at falconindy.com
Sun Aug 14 18:23:25 UTC 2016
On Sun, Aug 14, 2016 at 08:10:52PM +0200, Gordian Edenhofer wrote:
> * move the actual assembly process into its own function
> * allow for packaging multiple packages with one command
> * handle SIGHUP SIGINT SIGTERM and remove working dirs accordingly
> * add some comments
Can't this be done already by running multiple instances of bacman? In
other words, what's gained by adding this complexity to bacman itself?
Bonus, if you're on a system that has bash 4.3, you can do better than
the parallelize function you wrote and avoiding a tight loop and the CPU
it'll eat up:
pbacman() {
local max_jobs=$1; shift
for arg; do
while $(( $(jobs -p | wc -l) > max_jobs )); do
wait -n
done
bacman "$arg" &
done
wait
}
We can't use this in bacman without bumping the bash version requirement
which probably isn't something the pacman maintainers are willing to do.
>
> Signed-off-by: Gordian Edenhofer <gordian.edenhofer at gmail.com>
> ---
> contrib/bacman.sh.in | 460 ++++++++++++++++++++++++++-------------------------
> 1 file changed, 238 insertions(+), 222 deletions(-)
> mode change 100644 => 100755 contrib/bacman.sh.in
>
> diff --git a/contrib/bacman.sh.in b/contrib/bacman.sh.in
> old mode 100644
> new mode 100755
> index a611c1a..4cd78e4
> --- a/contrib/bacman.sh.in
> +++ b/contrib/bacman.sh.in
> @@ -33,9 +33,18 @@ ARGS=("$@")
>
> m4_include(../scripts/library/output_format.sh)
>
> -#
> -# User Friendliness
> -#
> +# Lazy recursive clean up of temporary dirs
> +work_dir_root="${TMPDIR:-/tmp}/bacman"
> +clean_up() {
> + rm -r "${work_dir_root}".*
> + echo
> + exit
> +}
> +
> +# Trap termination signals
> +trap clean_up SIGHUP SIGINT SIGTERM
> +
> +# Print usage information
> usage() {
> echo "${myname} (pacman) v${myver}"
> echo
> @@ -46,12 +55,14 @@ usage() {
> echo "Example: ${myname} linux-headers"
> }
>
> +# Print version information
> version() {
> printf "%s %s\n" "$myname" "$myver"
> echo 'Copyright (C) 2008 locci <carlocci_at_gmail_dot_com>'
> echo 'Copyright (C) 2008-2016 Pacman Development Team <pacman-dev at archlinux.org>'
> }
>
> +# Check for specified arguments
> while [[ ! -z $1 ]]; do
> if [[ $1 == "--nocolor" ]]; then
> USE_COLOR='n'
> @@ -64,13 +75,16 @@ while [[ ! -z $1 ]]; do
> fi
> done
>
> +# Configure colored output
> m4_include(../scripts/library/term_colors.sh)
>
> -if (( $# != 1 )); then
> +# Break if no argument was given
> +if (( $# < 1 )); then
> usage
> exit 1
> fi
>
> +# Print usage or version if requested
> if [[ $1 = -@(h|-help) ]]; then
> usage
> exit 0
> @@ -79,9 +93,7 @@ elif [[ $1 = -@(V|-version) ]]; then
> exit 0
> fi
>
> -#
> -# Fakeroot support
> -#
> +# Run in fakeroot if EUID is not root
> if (( EUID )); then
> if [[ -f /usr/bin/fakeroot ]]; then
> msg "Entering fakeroot environment"
> @@ -94,264 +106,268 @@ if (( EUID )); then
> fi
> fi
>
> -#
> -# Setting environmental variables
> -#
> +# Source environmental variables and specify fallbacks
> if [[ ! -r @sysconfdir@/pacman.conf ]]; then
> error "unable to read @sysconfdir@/pacman.conf"
> exit 1
> fi
> -
> eval $(awk '/DBPath/ {print $1$2$3}' @sysconfdir@/pacman.conf)
> pac_db="${DBPath:- at localstatedir@/lib/pacman/}/local"
> -
> if [[ ! -r @sysconfdir@/makepkg.conf ]]; then
> error "unable to read @sysconfdir@/makepkg.conf"
> exit 1
> fi
> -
> source "@sysconfdir@/makepkg.conf"
> if [[ -r ~/.makepkg.conf ]]; then
> source ~/.makepkg.conf
> fi
> -
> pkg_dest="${PKGDEST:-$PWD}"
> pkg_pkger=${PACKAGER:-'Unknown Packager'}
>
> -pkg_name="$1"
> -pkg_dir=("$pac_db/$pkg_name"-+([^-])-+([^-]))
> -pkg_namver=("${pkg_dir[@]##*/}")
> -
> -#
> -# Checks everything is in place
> -#
> +# Check for an existing database
> if [[ ! -d $pac_db ]]; then
> error "pacman database directory ${pac_db} not found"
> exit 1
> fi
>
> -if (( ${#pkg_dir[@]} != 1 )); then
> - error "%d entries for package %s found in pacman database" \
> - ${#pkg_dir[@]} "${pkg_name}"
> - msg2 "%s" "${pkg_dir[@]}"
> - exit 1
> -fi
> -
> -if [[ ! -d $pkg_dir ]]; then
> - error "package %s is found in pacman database," "${pkg_name}"
> - plain " but '%s' is not a directory" "${pkg_dir}"
> - exit 1
> -fi
> -
> -#
> -# Begin
> -#
> -msg "Package: ${pkg_namver}"
> -work_dir=$(mktemp -d "${TMPDIR:-/tmp}/bacman.XXXXXXXXXX")
> -cd "$work_dir" || exit 1
> -
> -#
> -# File copying
> -#
> -msg2 "Copying package files..."
> -
> -while read i; do
> - if [[ -z $i ]]; then
> - continue
> +# Assemble a single package: $1 = pkgname
> +fakebuild() {
> + pkg_name="$1"
> + pkg_dir=("$pac_db/$pkg_name"-+([^-])-+([^-]))
> + pkg_namver=("${pkg_dir[@]##*/}")
> +
> + # Checks database for specified package
> + if (( ${#pkg_dir[@]} != 1 )); then
> + error "%d entries for package %s found in pacman database" \
> + ${#pkg_dir[@]} "${pkg_name}"
> + msg2 "%s" "${pkg_dir[@]}"
> + exit 1
> fi
> -
> - if [[ $i == %+([A-Z])% ]]; then
> - current=$i
> - continue
> + if [[ ! -d $pkg_dir ]]; then
> + error "package %s is found in pacman database," "${pkg_name}"
> + plain " but '%s' is not a directory" "${pkg_dir}"
> + exit 1
> fi
>
> - case "$current" in
> - %FILES%)
> - local_file="/$i"
> - package_file="$work_dir/$i"
> + # Create working directory
> + msg "Package: ${pkg_namver}"
> + work_dir=$(mktemp -d "${work_dir_root}.XXXXXXXXXX")
> + cd "$work_dir" || exit 1
>
> - if [[ ! -e $local_file ]]; then
> - warning "package file $local_file is missing"
> - continue
> - fi
> - ;;
> -
> - %BACKUP%)
> - # Get the MD5 checksum.
> - original_md5="${i##*$'\t'}"
> - # Strip the md5sum after the tab.
> - i="${i%$'\t'*}"
> - local_file="/$i.pacnew"
> - package_file="$work_dir/$i"
> -
> - # Include unmodified .pacnew files.
> - local_md5="$(md5sum "$local_file" | cut -d' ' -f1)"
> - if [[ $INCLUDE_PACNEW == 'n' ]] \
> - || [[ ! -e $local_file ]] \
> - || [[ $local_md5 != $original_md5 ]]; then
> - # Warn about modified files.
> - local_md5="$(md5sum "/$i" | cut -d' ' -f1)"
> - if [[ $local_md5 != $original_md5 ]]; then
> - warning "package file /$i has been modified"
> - fi
> - # Let the normal file be included in the %FILES% list.
> - continue
> - fi
> - ;;
> + # Assemble list of files which belong to the package and tar them
> + msg2 "Copying package files..."
> + while read i; do
> + if [[ -z $i ]]; then
> + continue
> + fi
>
> - *)
> + if [[ $i == %+([A-Z])% ]]; then
> + current=$i
> continue
> - ;;
> - esac
> + fi
>
> - ret=0
> - bsdtar -cnf - -s'/.pacnew$//' "$local_file" 2> /dev/null | bsdtar -xpf - 2> /dev/null
> + case "$current" in
> + %FILES%)
> + local_file="/$i"
> + package_file="$work_dir/$i"
> +
> + if [[ ! -e $local_file ]]; then
> + warning "package file $local_file is missing"
> + continue
> + fi
> + ;;
> +
> + %BACKUP%)
> + # Get the MD5 checksum.
> + original_md5="${i##*$'\t'}"
> + # Strip the md5sum after the tab.
> + i="${i%$'\t'*}"
> + local_file="/$i.pacnew"
> + package_file="$work_dir/$i"
> +
> + # Include unmodified .pacnew files.
> + local_md5="$(md5sum "$local_file" | cut -d' ' -f1)"
> + if [[ $INCLUDE_PACNEW == 'n' ]] \
> + || [[ ! -e $local_file ]] \
> + || [[ $local_md5 != $original_md5 ]]; then
> + # Warn about modified files.
> + local_md5="$(md5sum "/$i" | cut -d' ' -f1)"
> + if [[ $local_md5 != $original_md5 ]]; then
> + warning "package file /$i has been modified"
> + fi
> + # Let the normal file be included in the %FILES% list.
> + continue
> + fi
> + ;;
>
> - # Workaround to bsdtar not reporting a missing file as an error
> - if ! [[ -e $package_file || -L $package_file ]]; then
> - error "unable to add $local_file to the package"
> - plain " If your user does not have permission to read this file, then"
> - plain " you will need to run $myname as root."
> + *)
> + continue
> + ;;
> + esac
> +
> + # Tar files
> + ret=0
> + bsdtar -cnf - -s'/.pacnew$//' "$local_file" 2> /dev/null | bsdtar -xpf - 2> /dev/null
> + # Workaround to bsdtar not reporting a missing file as an error
> + if ! [[ -e $package_file || -L $package_file ]]; then
> + error "unable to add $local_file to the package"
> + plain " If your user does not have permission to read this file, then"
> + plain " you will need to run $myname as root."
> + rm -rf "$work_dir"
> + exit 1
> + fi
> + done < "$pkg_dir"/files
> +
> + ret=$?
> + if (( ret )); then
> rm -rf "$work_dir"
> exit 1
> fi
> -done < "$pkg_dir"/files
>
> -ret=$?
> -if (( ret )); then
> - rm -rf "$work_dir"
> - exit 1
> -fi
> + # Calculate package size
> + pkg_size=$(du -sk | awk '{print $1 * 1024}')
>
> -pkg_size=$(du -sk | awk '{print $1 * 1024}')
> -
> -#
> -# .PKGINFO stuff
> -# TODO adopt makepkg's write_pkginfo() into this or scripts/library
> -#
> -msg2 "Generating .PKGINFO metadata..."
> -echo "# Generated by $myname $myver" > .PKGINFO
> -if [[ $INFAKEROOT == "1" ]]; then
> - echo "# Using $(fakeroot -v)" >> .PKGINFO
> -fi
> -echo "# $(LC_ALL=C date)" >> .PKGINFO
> -echo "#" >> .PKGINFO
> -
> -while read i; do
> - if [[ -z $i ]]; then
> - continue;
> + # Reconstruct .PKGINFO from database
> + # TODO adopt makepkg's write_pkginfo() into this or scripts/library
> + msg2 "Generating .PKGINFO metadata..."
> + echo "# Generated by $myname $myver" > .PKGINFO
> + if [[ $INFAKEROOT == "1" ]]; then
> + echo "# Using $(fakeroot -v)" >> .PKGINFO
> fi
> -
> - if [[ $i == %+([A-Z])% ]]; then
> - current=$i
> - continue
> + echo "# $(LC_ALL=C date)" >> .PKGINFO
> + echo "#" >> .PKGINFO
> + while read i; do
> + if [[ -z $i ]]; then
> + continue;
> + fi
> + if [[ $i == %+([A-Z])% ]]; then
> + current=$i
> + continue
> + fi
> +
> + case "$current" in
> + # desc
> + %NAME%)
> + echo "pkgname = $i" >> .PKGINFO
> + ;;
> + %VERSION%)
> + echo "pkgver = $i" >> .PKGINFO
> + ;;
> + %DESC%)
> + echo "pkgdesc = $i" >> .PKGINFO
> + ;;
> + %URL%)
> + echo "url = $i" >> .PKGINFO
> + ;;
> + %LICENSE%)
> + echo "license = $i" >> .PKGINFO
> + ;;
> + %ARCH%)
> + echo "arch = $i" >> .PKGINFO
> + pkg_arch="$i"
> + ;;
> + %BUILDDATE%)
> + echo "builddate = $(date -u "+%s")" >> .PKGINFO
> + ;;
> + %PACKAGER%)
> + echo "packager = $pkg_pkger" >> .PKGINFO
> + ;;
> + %SIZE%)
> + echo "size = $pkg_size" >> .PKGINFO
> + ;;
> + %GROUPS%)
> + echo "group = $i" >> .PKGINFO
> + ;;
> + %REPLACES%)
> + echo "replaces = $i" >> .PKGINFO
> + ;;
> + %DEPENDS%)
> + echo "depend = $i" >> .PKGINFO
> + ;;
> + %OPTDEPENDS%)
> + echo "optdepend = $i" >> .PKGINFO
> + ;;
> + %CONFLICTS%)
> + echo "conflict = $i" >> .PKGINFO
> + ;;
> + %PROVIDES%)
> + echo "provides = $i" >> .PKGINFO
> + ;;
> + %BACKUP%)
> + # Strip the md5sum after the tab
> + echo "backup = ${i%%$'\t'*}" >> .PKGINFO
> + ;;
> + esac
> + done < <(cat "$pkg_dir"/{desc,files})
> +
> + comp_files=".PKGINFO"
> +
> + # Add instal file if present
> + if [[ -f $pkg_dir/install ]]; then
> + cp "$pkg_dir/install" "$work_dir/.INSTALL"
> + comp_files+=" .INSTALL"
> + fi
> + if [[ -f $pkg_dir/changelog ]]; then
> + cp "$pkg_dir/changelog" "$work_dir/.CHANGELOG"
> + comp_files+=" .CHANGELOG"
> fi
>
> - case "$current" in
> - # desc
> - %NAME%)
> - echo "pkgname = $i" >> .PKGINFO
> - ;;
> - %VERSION%)
> - echo "pkgver = $i" >> .PKGINFO
> - ;;
> - %DESC%)
> - echo "pkgdesc = $i" >> .PKGINFO
> - ;;
> - %URL%)
> - echo "url = $i" >> .PKGINFO
> - ;;
> - %LICENSE%)
> - echo "license = $i" >> .PKGINFO
> - ;;
> - %ARCH%)
> - echo "arch = $i" >> .PKGINFO
> - pkg_arch="$i"
> - ;;
> - %BUILDDATE%)
> - echo "builddate = $(date -u "+%s")" >> .PKGINFO
> - ;;
> - %PACKAGER%)
> - echo "packager = $pkg_pkger" >> .PKGINFO
> - ;;
> - %SIZE%)
> - echo "size = $pkg_size" >> .PKGINFO
> - ;;
> - %GROUPS%)
> - echo "group = $i" >> .PKGINFO
> - ;;
> - %REPLACES%)
> - echo "replaces = $i" >> .PKGINFO
> - ;;
> - %DEPENDS%)
> - echo "depend = $i" >> .PKGINFO
> - ;;
> - %OPTDEPENDS%)
> - echo "optdepend = $i" >> .PKGINFO
> - ;;
> - %CONFLICTS%)
> - echo "conflict = $i" >> .PKGINFO
> - ;;
> - %PROVIDES%)
> - echo "provides = $i" >> .PKGINFO
> - ;;
> -
> - # files
> - %BACKUP%)
> - # Strip the md5sum after the tab
> - echo "backup = ${i%%$'\t'*}" >> .PKGINFO
> - ;;
> - esac
> -done < <(cat "$pkg_dir"/{desc,files})
> -
> -comp_files=".PKGINFO"
> -
> -if [[ -f $pkg_dir/install ]]; then
> - cp "$pkg_dir/install" "$work_dir/.INSTALL"
> - comp_files+=" .INSTALL"
> -fi
> -if [[ -f $pkg_dir/changelog ]]; then
> - cp "$pkg_dir/changelog" "$work_dir/.CHANGELOG"
> - comp_files+=" .CHANGELOG"
> -fi
> + # Fixes owner:group and permissions for .PKGINFO, .CHANGELOG, .INSTALL
> + chown root:root "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL} 2> /dev/null
> + chmod 644 "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL} 2> /dev/null
>
> -#
> -# Fixes owner:group and permissions for .PKGINFO, .CHANGELOG, .INSTALL
> -#
> -chown root:root "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL} 2> /dev/null
> -chmod 644 "$work_dir"/{.PKGINFO,.CHANGELOG,.INSTALL} 2> /dev/null
> + # Generate the package
> + msg2 "Generating the package..."
>
> -#
> -# Generate the package
> -#
> -msg2 "Generating the package..."
> -
> -pkg_file="$pkg_dest/$pkg_namver-$pkg_arch${PKGEXT}"
> -ret=0
> -
> -# TODO: Maybe this can be set globally for robustness
> -shopt -s -o pipefail
> -bsdtar -cf - $comp_files * |
> -case "$PKGEXT" in
> - *tar.gz) gzip -c -f -n ;;
> - *tar.bz2) bzip2 -c -f ;;
> - *tar.xz) xz -c -z - ;;
> - *tar.Z) compress -c -f ;;
> - *tar) cat ;;
> - *) warning "'%s' is not a valid archive extension." \
> - "$PKGEXT"; cat ;;
> -esac > "${pkg_file}"; ret=$?
> -
> -if (( ret )); then
> - error "Unable to write package to $pkg_dest"
> - plain " Maybe the disk is full or you do not have write access"
> + pkg_file="$pkg_dest/$pkg_namver-$pkg_arch${PKGEXT}"
> + ret=0
> +
> + # TODO: Maybe this can be set globally for robustness
> + shopt -s -o pipefail
> + bsdtar -cf - $comp_files * |
> + case "$PKGEXT" in
> + *tar.gz) gzip -c -f -n ;;
> + *tar.bz2) bzip2 -c -f ;;
> + *tar.xz) xz -c -z - ;;
> + *tar.Z) compress -c -f ;;
> + *tar) cat ;;
> + *) warning "'%s' is not a valid archive extension." \
> + "$PKGEXT"; cat ;;
> + esac > "${pkg_file}"; ret=$?
> +
> + # Move compressed package to destination
> + if (( ret )); then
> + error "Unable to write package to $pkg_dest"
> + plain " Maybe the disk is full or you do not have write access"
> + rm -rf "$work_dir"
> + exit 1
> + fi
> +
> + # Clean up working directory
> rm -rf "$work_dir"
> - exit 1
> -fi
> + msg "Done."
> +}
>
> -rm -rf "$work_dir"
> +# Run fakebuild in parralel with at maximum $MAX_JOBS jobs
> +# By default only run one job
> +MAX_JOBS=${MAX_JOBS:-1}
> +parallelize() {
> + while [[ $# -gt 0 ]] ; do
> + job_count=($(jobs -p))
> + if [[ ${#job_count[@]} -lt $MAX_JOBS ]] ; then
> + fakebuild "$1" &
> + shift
> + fi
> + done
> + wait
> +}
>
> -msg "Done."
> +# Initiate assembly function
> +if [[ $MAX_JOBS -gt "1" ]]; then
> + parallelize "$@"
> +else
> + for PKG in $@; do fakebuild $PKG; done
> +fi
>
> exit 0
>
> --
> 2.9.2
More information about the pacman-dev
mailing list