gentoo auto-resync : 28:09:2022 - 03:11:19

4 files changed, 422 insertions, 55 deletions

diff --git a/eclass/Manifest.gz b/eclass/Manifest.gz
index 9a9f1ef321f4..45c6c8ad25d7 100644
--- a/eclass/Manifest.gz
+++ b/eclass/Manifest.gz
diff --git a/eclass/tests/tests-common.sh b/eclass/tests/tests-common.sh
index a677842b6ac5..45b1e20b933a 100644
--- a/eclass/tests/tests-common.sh
+++ b/eclass/tests/tests-common.sh
@@ -60,6 +60,13 @@ die() {
 	exit 1
 }
 
+assert() {
+	local x pipestatus=${PIPESTATUS[*]}
+	for x in ${pipestatus} ; do
+		[[ ${x} -eq 0 ]] || die "$@"
+	done
+}
+
 has_version() {
 	while [[ $1 == -* ]]; do
 		shift
diff --git a/eclass/tests/unpacker.sh b/eclass/tests/unpacker.sh
new file mode 100755
index 000000000000..b15953966f65
--- /dev/null
+++ b/eclass/tests/unpacker.sh
@@ -0,0 +1,291 @@
+#!/usr/bin/env bash
+# Copyright 2022 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+source tests-common.sh || exit
+
+inherit unpacker
+
+# silence the output
+unpack_banner() { :; }
+
+TESTFILE=test.in
+TESTDIR=$(mktemp -d || die)
+trap 'cd - >/dev/null && rm -r "${TESTDIR}"' EXIT
+
+# prepare some test data
+# NB: we need something "compressible", as compress(1) will return
+# an error if the file "is larger than before compression"
+cp ../unpacker.eclass "${TESTDIR}/${TESTFILE}" || die
+cd "${TESTDIR}" || die
+
+test_unpack() {
+	local archive=${1}
+	local unpacked=${2}
+	local deps=${3}
+	local packcmd=${4}
+
+	local x
+	for x in ${deps}; do
+		if ! type "${x}" &>/dev/null; then
+			ewarn "Skipping ${archive}, tool ${x} not found"
+			return
+		fi
+	done
+
+	rm -rf testdir || die
+	mkdir -p testdir || die
+
+	tbegin "unpacking ${archive}"
+	eval "${packcmd}"
+	assert "packing ${archive} failed"
+	cd testdir || die
+	local out
+	out=$(
+		_unpacker "../${archive}" 2>&1
+	)
+	ret=$?
+	if [[ ${ret} -eq 0 ]]; then
+		if [[ ! -f ${unpacked} ]]; then
+			eerror "${unpacked} not found after unpacking"
+			ret=1
+		elif ! diff -u "${unpacked}" "../${TESTFILE}"; then
+			eerror "${unpacked} different than input"
+			ret=1
+		fi
+	fi
+	[[ ${ret} -ne 0 ]] && echo "${out}" >&2
+	tend ${ret}
+
+	cd .. || die
+	rm -f "${archive}" || die
+}
+
+test_compressed_file() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test${suffix}" test "${tool}" \
+		"${tool} -c \${TESTFILE} > \${archive}"
+}
+
+test_compressed_file_multistream() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test+multistream${suffix}" "test+multistream" "${tool}" \
+		"head -n 300 \${TESTFILE} | ${tool} -c > \${archive} &&
+		tail -n +301 \${TESTFILE} | ${tool} -c >> \${archive}"
+}
+
+test_compressed_file_with_junk() {
+	local suffix=${1}
+	local tool=${2}
+	local flag=${3}
+
+	test_unpack "test+junk${suffix}" "test+junk" "${tool}" \
+		"${tool} -c \${TESTFILE} > \${archive} && cat test.in >> \${archive}"
+}
+
+test_compressed_tar() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test${suffix}" test.in "tar ${tool}" \
+		"tar -c \${TESTFILE} | ${tool} -c > \${archive}"
+}
+
+test_compressed_cpio() {
+	local suffix=${1}
+	local tool=${2}
+
+	test_unpack "test${suffix}" test.in "cpio ${tool}" \
+		"cpio -o --quiet <<<\${TESTFILE} | ${tool} -c > \${archive}"
+}
+
+create_deb() {
+	local suffix=${1}
+	local tool=${2}
+	local archive=${3}
+	local infile=${4}
+
+	echo 2.0 > debian-binary || die
+	: > control || die
+	tar -cf control.tar control || die
+	tar -c "${infile}" | ${tool} > "data.tar${suffix}"
+	assert "packing data.tar${suffix} failed"
+	ar r "${archive}" debian-binary control.tar "data.tar${suffix}" \
+		2>/dev/null || die
+	rm -f control control.tar "data.tar${suffix}" debian-binary || die
+}
+
+test_deb() {
+	local suffix=${1}
+	local tool=${2}
+	local tool_cmd
+
+	if [[ -n ${tool} ]]; then
+		tool_cmd="${tool} -c"
+	else
+		tool_cmd=cat
+	fi
+
+	test_unpack "test-${tool}_1.2.3_noarch.deb" test.in "ar tar ${tool}" \
+		"create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
+	# also test with the handwoven implementation used on Prefix
+	EPREFIX=/foo \
+	test_unpack "test_pfx-${tool}_1.2.3_noarch.deb" test.in "ar tar ${tool}" \
+		"create_deb '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
+}
+
+create_gpkg() {
+	local suffix=${1}
+	local tool=${2}
+	local archive=${3}
+	local infile=${4}
+	local gpkg_dir=${archive%.gpkg.tar}
+
+	mkdir image metadata "${gpkg_dir}" || die
+	cp "${infile}" image/ || die
+	tar -c metadata | ${tool} > "${gpkg_dir}/metadata.tar${suffix}"
+	assert "packing metadata.tar${suffix} failed"
+	: > "${gpkg_dir}/metadata.tar${suffix}.sig" || die
+	tar -c image | ${tool} > "${gpkg_dir}/image.tar${suffix}"
+	assert "packing image.tar${suffix} failed"
+	: > "${gpkg_dir}/image.tar${suffix}.sig" || die
+	: > "${gpkg_dir}"/gpkg-1 || die
+	tar -cf "${archive}" --format=ustar \
+		"${gpkg_dir}"/{gpkg-1,{metadata,image}.tar"${suffix}"} || die
+	rm -r image metadata "${gpkg_dir}" || die
+}
+
+test_gpkg() {
+	local suffix=${1}
+	local tool=${2}
+	local tool_cmd
+
+	if [[ -n ${tool} ]]; then
+		tool_cmd="${tool} -c"
+	else
+		tool_cmd=cat
+	fi
+
+	test_unpack "test-${tool}-1.2.3-1.gpkg.tar" \
+		"test-${tool}-1.2.3-1/image/test.in" "tar ${tool}" \
+		"create_gpkg '${suffix}' '${tool_cmd}' \${archive} \${TESTFILE}"
+}
+
+test_reject_junk() {
+	local suffix=${1}
+	local archive=test${1}
+
+	rm -rf testdir || die
+	mkdir -p testdir || die
+
+	tbegin "rejecting junk named ${archive}"
+	cat test.in >> "${archive}" || die
+	cd testdir || die
+	(
+		# some decompressors (e.g. cpio) are very verbose about junk
+		_unpacker "../${archive}" &>/dev/null
+	)
+	[[ $? -ne 0 ]]
+	ret=$?
+	tend ${ret}
+
+	cd .. || die
+	rm -f "${archive}" || die
+}
+
+test_compressed_file .bz2 bzip2
+test_compressed_file .Z compress
+test_compressed_file .gz gzip
+test_compressed_file .lzma lzma
+test_compressed_file .xz xz
+test_compressed_file .lz lzip
+test_compressed_file .zst zstd
+test_compressed_file .lz4 lz4
+test_compressed_file .lzo lzop
+
+test_compressed_file_multistream .bz2 bzip2
+test_compressed_file_multistream .gz gzip
+test_compressed_file_multistream .xz xz
+test_compressed_file_multistream .lz lzip
+test_compressed_file_multistream .zst zstd
+
+test_compressed_file_with_junk .bz2 bzip2
+test_compressed_file_with_junk .lz lzip
+
+test_unpack test.tar test.in tar 'tar -cf ${archive} ${TESTFILE}'
+test_compressed_tar .tar.bz2 bzip2
+test_compressed_tar .tbz bzip2
+test_compressed_tar .tbz2 bzip2
+test_compressed_tar .tar.Z compress
+test_compressed_tar .tar.gz gzip
+test_compressed_tar .tgz gzip
+test_compressed_tar .tar.lzma lzma
+test_compressed_tar .tar.xz xz
+test_compressed_tar .txz xz
+test_compressed_tar .tar.lz lzip
+test_compressed_tar .tar.zst zstd
+test_compressed_tar .tar.lz4 lz4
+test_compressed_tar .tar.lzo lzop
+
+test_unpack test.cpio test.in cpio 'cpio -o --quiet <<<${TESTFILE} > ${archive}'
+test_compressed_cpio .cpio.bz2 bzip2
+test_compressed_cpio .cpio.Z compress
+test_compressed_cpio .cpio.gz gzip
+test_compressed_cpio .cpio.lzma lzma
+test_compressed_cpio .cpio.xz xz
+test_compressed_cpio .cpio.lz lzip
+test_compressed_cpio .cpio.zst zstd
+test_compressed_cpio .cpio.lz4 lz4
+test_compressed_cpio .cpio.lzo lzop
+
+test_deb
+test_deb .gz gzip
+test_deb .xz xz
+test_deb .bz2 bzip2
+test_deb .lzma lzma
+
+test_gpkg
+test_gpkg .gz gzip
+test_gpkg .bz2 bzip2
+test_gpkg .lz4 lz4
+test_gpkg .lz lzip
+test_gpkg .lzma lzma
+test_gpkg .lzo lzop
+test_gpkg .xz xz
+test_gpkg .zst zstd
+
+test_unpack test.zip test.in zip 'zip -q ${archive} ${TESTFILE}'
+# test handling non-adjusted zip with junk prepended
+test_unpack test.zip test.in zip \
+	'zip -q testdir/tmp.zip ${TESTFILE} && cat test.in testdir/tmp.zip > ${archive}'
+test_unpack test.7z test.in 7z '7z -bso0 a ${archive} ${TESTFILE}'
+test_unpack test.lha test.in lha 'lha a -q ${archive} ${TESTFILE}'
+test_unpack test.lzh test.in lha 'lha a -q ${archive} ${TESTFILE}'
+test_unpack test.rar test.in rar 'rar -idq a ${archive} ${TESTFILE}'
+
+# TODO: .run/.sh/.bin
+
+test_reject_junk .bz2
+test_reject_junk .Z
+test_reject_junk .gz
+test_reject_junk .lzma
+test_reject_junk .xz
+test_reject_junk .lz
+test_reject_junk .zst
+test_reject_junk .tar
+test_reject_junk .cpio
+test_reject_junk .gpkg.tar
+test_reject_junk .deb
+test_reject_junk .zip
+test_reject_junk .7z
+test_reject_junk .rar
+test_reject_junk .lha
+test_reject_junk .lzh
+
+texit
diff --git a/eclass/unpacker.eclass b/eclass/unpacker.eclass
index f6e83c53bf23..3d23151b636e 100644
--- a/eclass/unpacker.eclass
+++ b/eclass/unpacker.eclass
@@ -4,7 +4,7 @@
 # @ECLASS: unpacker.eclass
 # @MAINTAINER:
 # base-system@gentoo.org
-# @SUPPORTED_EAPIS: 5 6 7 8
+# @SUPPORTED_EAPIS: 6 7 8
 # @BLURB: helpers for extraneous file formats and consistent behavior across EAPIs
 # @DESCRIPTION:
 # Some extraneous file formats are not part of PMS, or are only in certain
@@ -16,21 +16,22 @@
 #  - support partial unpacks?
 
 case ${EAPI:-0} in
-	[5678]) ;;
+	[678]) ;;
 	*) die "${ECLASS}: EAPI ${EAPI:-0} not supported" ;;
 esac
 
 if [[ -z ${_UNPACKER_ECLASS} ]]; then
 _UNPACKER_ECLASS=1
 
-inherit toolchain-funcs
+inherit multiprocessing toolchain-funcs
 
 # @ECLASS_VARIABLE: UNPACKER_BZ2
 # @USER_VARIABLE
 # @DEFAULT_UNSET
 # @DESCRIPTION:
 # Utility to use to decompress bzip2 files.  Will dynamically pick between
-# `pbzip2` and `bzip2`.  Make sure your choice accepts the "-dc" options.
+# `lbzip2`, `pbzip2` and `bzip2`.  Make sure your choice accepts the "-dc"
+# options.
 # Note: this is meant for users to set, not ebuilds.
 
 # @ECLASS_VARIABLE: UNPACKER_LZIP
@@ -272,31 +273,39 @@ unpack_deb() {
 
 	unpack_banner "${deb}"
 
-	# on AIX ar doesn't work out as their ar used a different format
-	# from what GNU ar (and thus what .deb files) produce
-	if [[ -n ${EPREFIX} ]] ; then
-		{
-			read # global header
-			[[ ${REPLY} = "!<arch>" ]] || die "${deb} does not seem to be a deb archive"
-			local f timestamp uid gid mode size magic
-			while read f timestamp uid gid mode size magic ; do
-				[[ -n ${f} && -n ${size} ]] || continue # ignore empty lines
-				if [[ ${f} = "data.tar"* ]] ; then
-					head -c "${size}" > "${f}"
-				else
-					head -c "${size}" > /dev/null # trash it
-				fi
-			done
-		} < "${deb}"
-	else
-		$(tc-getBUILD_AR) x "${deb}" || die
-	fi
-
-	unpacker ./data.tar*
-
-	# Clean things up #458658.  No one seems to actually care about
-	# these, so wait until someone requests to do something else ...
-	rm -f debian-binary {control,data}.tar*
+	{
+		# on AIX ar doesn't work out as their ar used a different format
+		# from what GNU ar (and thus what .deb files) produce
+		if [[ -n ${EPREFIX} ]] ; then
+			{
+				read # global header
+				[[ ${REPLY} = "!<arch>" ]] || die "${deb} does not seem to be a deb archive"
+				local f timestamp uid gid mode size magic
+				while read f timestamp uid gid mode size magic ; do
+					[[ -n ${f} && -n ${size} ]] || continue # ignore empty lines
+					# GNU ar uses / as filename terminator (and .deb permits that)
+					f=${f%/}
+					if [[ ${f} = "data.tar"* ]] ; then
+						local decomp=$(_unpacker_get_decompressor "${f}")
+						head -c "${size}" | ${decomp:-cat}
+						assert "unpacking ${f} from ${deb} failed"
+						break
+					else
+						head -c "${size}" > /dev/null # trash it
+					fi
+				done
+			} < "${deb}"
+		else
+			local f=$(
+				$(tc-getBUILD_AR) t "${deb}" | grep ^data.tar
+				assert "data not found in ${deb}"
+			)
+			local decomp=$(_unpacker_get_decompressor "${f}")
+			$(tc-getBUILD_AR) p "${deb}" "${f}" | ${decomp:-cat}
+			assert "unpacking ${f} from ${deb} failed"
+		fi
+	} | tar --no-same-owner -x
+	assert "unpacking ${deb} failed"
 }
 
 # @FUNCTION: unpack_cpio
@@ -344,8 +353,11 @@ unpack_7z() {
 
 	local p7z=$(find_unpackable_file "$1")
 	unpack_banner "${p7z}"
-	local output="$(7z x -y "${p7z}")"
 
+	# warning: putting local and command substitution in a single call
+	# discards the exit status!
+	local output
+	output="$(7z x -y "${p7z}")"
 	if [ $? -ne 0 ]; then
 		echo "${output}" >&2
 		die "unpacking ${p7z} failed (arch=unpack_7z)"
@@ -376,6 +388,73 @@ unpack_lha() {
 	lha xfq "${lha}" || die "unpacking ${lha} failed (arch=unpack_lha)"
 }
 
+# @FUNCTION: _unpacker_get_decompressor
+# @INTERNAL
+# @USAGE: <filename>
+# @DESCRIPTION:
+# Get decompressor command for specified filename.
+_unpacker_get_decompressor() {
+	case ${1} in
+	*.bz2|*.tbz|*.tbz2)
+		local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(
+			type -P lbzip2 || type -P pbzip2 || type -P bzip2
+		)}
+		local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${bzcmd} -d}
+		: ${UNPACKER_BZ2:=${bzuncmd}}
+		echo "${UNPACKER_BZ2} -c"
+		;;
+	*.z|*.gz|*.tgz)
+		echo "gzip -dc" ;;
+	*.lzma|*.xz|*.txz)
+		echo "xz -T$(makeopts_jobs) -dc" ;;
+	*.lz)
+		: ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)}
+		echo "${UNPACKER_LZIP} -dc" ;;
+	*.zst)
+		echo "zstd -dc" ;;
+	*.lz4)
+		echo "lz4 -dc" ;;
+	*.lzo)
+		echo "lzop -dc" ;;
+	esac
+}
+
+# @FUNCTION: unpack_gpkg
+# @USAGE: <gpkg file>
+# @DESCRIPTION:
+# Unpack the image subarchive of a GPKG package on-the-fly, preserving
+# the original directory structure (i.e. into <gpkg-dir>/image).
+unpack_gpkg() {
+	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"
+
+	local gpkg=$(find_unpackable_file "$1")
+	unpack_banner "${gpkg}"
+
+	local l images=()
+	while read -r l; do
+		case ${l} in
+			*/image.tar*.sig)
+				;;
+			*/image.tar*)
+				images+=( "${l}" )
+				;;
+		esac
+	done < <(tar -tf "${gpkg}" || die "unable to list ${gpkg}")
+
+	if [[ ${#images[@]} -eq 0 ]]; then
+		die "No image.tar found in ${gpkg}"
+	elif [[ ${#images[@]} -gt 1 ]]; then
+		die "More than one image.tar found in ${gpkg}"
+	fi
+
+	local decomp=$(_unpacker_get_decompressor "${images[0]}")
+	local dirname=${images[0]%/*}
+	mkdir -p "${dirname}" || die
+	tar -xOf "${gpkg}" "${images[0]}" | ${decomp:-cat} |
+		tar --no-same-owner -xC "${dirname}"
+	assert "Unpacking ${gpkg} failed"
+}
+
 # @FUNCTION: _unpacker
 # @USAGE: <one archive to unpack>
 # @INTERNAL
@@ -386,32 +465,17 @@ _unpacker() {
 	[[ $# -eq 1 ]] || die "Usage: ${FUNCNAME} <file>"
 
 	local a=$1
-	local m=$(echo "${a}" | tr '[:upper:]' '[:lower:]')
+	local m=${a,,}
 	a=$(find_unpackable_file "${a}")
 
 	# first figure out the decompression method
-	local comp=""
-	case ${m} in
-	*.bz2|*.tbz|*.tbz2)
-		local bzcmd=${PORTAGE_BZIP2_COMMAND:-$(type -P pbzip2 || type -P bzip2)}
-		local bzuncmd=${PORTAGE_BUNZIP2_COMMAND:-${bzcmd} -d}
-		: ${UNPACKER_BZ2:=${bzuncmd}}
-		comp="${UNPACKER_BZ2} -c"
-		;;
-	*.z|*.gz|*.tgz)
-		comp="gzip -dc" ;;
-	*.lzma|*.xz|*.txz)
-		comp="xz -dc" ;;
-	*.lz)
-		: ${UNPACKER_LZIP:=$(type -P plzip || type -P pdlzip || type -P lzip)}
-		comp="${UNPACKER_LZIP} -dc" ;;
-	*.zst)
-		comp="zstd -dfc" ;;
-	esac
+	local comp=$(_unpacker_get_decompressor "${m}")
 
 	# then figure out if there are any archiving aspects
 	local arch=""
 	case ${m} in
+	*.gpkg.tar)
+		arch="unpack_gpkg" ;;
 	*.tgz|*.tbz|*.tbz2|*.txz|*.tar.*|*.tar)
 		arch="tar --no-same-owner -xof" ;;
 	*.cpio.*|*.cpio)
@@ -437,13 +501,13 @@ _unpacker() {
 	esac
 
 	# 7z, rar and lha/lzh are handled by package manager in EAPI < 8
-	if [[ ${EAPI} != [567] ]]; then
+	if [[ ${EAPI} != [67] ]]; then
 		case ${m} in
 		*.7z)
 			arch="unpack_7z" ;;
-		*.rar|*.RAR)
+		*.rar)
 			arch="unpack_rar" ;;
-		*.LHA|*.LHa|*.lha|*.lzh)
+		*.lha|*.lzh)
 			arch="unpack_lha" ;;
 		esac
 	fi
@@ -506,10 +570,11 @@ unpacker_src_uri_depends() {
 	fi
 
 	for uri in "$@" ; do
-		case ${uri} in
+		local m=${uri,,}
+		case ${m} in
 		*.cpio.*|*.cpio)
 			d="app-arch/cpio" ;;
-		*.rar|*.RAR)
+		*.rar)
 			d="app-arch/unrar" ;;
 		*.7z)
 			d="app-arch/p7zip" ;;
@@ -521,8 +586,12 @@ unpacker_src_uri_depends() {
 			d="|| ( app-arch/plzip app-arch/pdlzip app-arch/lzip )" ;;
 		*.zst)
 			d="app-arch/zstd" ;;
-		*.LHA|*.LHa|*.lha|*.lzh)
+		*.lha|*.lzh)
 			d="app-arch/lha" ;;
+		*.lz4)
+			d="app-arch/lz4" ;;
+		*.lzo)
+			d="app-arch/lzop" ;;
 		esac
 		deps+=" ${d}"
 	done