Commit 32366a4ba2affb38c2a5535999daf1e67b37f392 - debuerreotype

New upstream version 0.2 Tianon Gravi 6 years ago

10 changed file(s) with 238 addition(s) and 72 deletion(s). Raw diff Collapse all Expand all

+10

-2

.travis.sh less more

3	3	epoch="$(TZ=UTC date --date "$TIMESTAMP" +%s)"
4	4	serial="$(TZ=UTC date --date "@$epoch" +%Y%m%d)"
5	5
	6	buildArgs=()
	7	if [ -n "${CODENAME:-}" ]; then
	8	buildArgs+=( '--codename-copy' )
	9	fi
	10	buildArgs+=( travis "$SUITE" "@$epoch" )
	11
	12	checkFile="travis/$serial/amd64/${CODENAME:-$SUITE}/rootfs.tar.xz"
	13
6	14	set -x
7	15
8	16	./scripts/debuerreotype-version
9		./build.sh travis "$SUITE" "@$epoch"
	17	./build.sh "${buildArgs[@]}"
10	18
11		real="$(sha256sum "travis/$serial/$SUITE-amd64.tar.xz" \| cut -d' ' -f1)"
	19	real="$(sha256sum "$checkFile" \| cut -d' ' -f1)"
12	20	[ -z "$SHA256" ] \|\| [ "$SHA256" = "$real" ]

-3

.travis.yml less more

1	1	services: docker
2	2
3	3	env:
4		- SUITE=jessie TIMESTAMP=2017-01-01T00:00:00Z SHA256=a674379d30cf457e49a909bb0e254ddcb1f72c27f45c68fe930668d6d9399232
5		- SUITE=stretch TIMESTAMP=2017-01-01T00:00:00Z SHA256=139ed970d52ef950c223f9ab325657eb93d0a93c7d6e2fc697fe7510e61760fa
6		- SUITE=sid TIMESTAMP=2017-01-01T00:00:00Z SHA256=b75b4496deb4d6cee32245e4125e7ef948b09afbeb1ef3b9669e56daf3e822a7
	4	- SUITE=jessie CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=a674379d30cf457e49a909bb0e254ddcb1f72c27f45c68fe930668d6d9399232
	5	- SUITE=stable CODENAME=jessie TIMESTAMP=2017-01-01T00:00:00Z SHA256=a674379d30cf457e49a909bb0e254ddcb1f72c27f45c68fe930668d6d9399232
	6	- SUITE=stretch CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=139ed970d52ef950c223f9ab325657eb93d0a93c7d6e2fc697fe7510e61760fa
	7	- SUITE=testing CODENAME=stretch TIMESTAMP=2017-01-01T00:00:00Z SHA256=139ed970d52ef950c223f9ab325657eb93d0a93c7d6e2fc697fe7510e61760fa
	8	- SUITE=sid CODENAME= TIMESTAMP=2017-01-01T00:00:00Z SHA256=b75b4496deb4d6cee32245e4125e7ef948b09afbeb1ef3b9669e56daf3e822a7
	9	- SUITE=unstable CODENAME=sid TIMESTAMP=2017-01-01T00:00:00Z SHA256=b75b4496deb4d6cee32245e4125e7ef948b09afbeb1ef3b9669e56daf3e822a7
7	10
8	11	script:
9	12	- travis_retry ./.travis.sh

-1

README.md less more

111	111	│ │ 0000070: 0000 0000 0000 0000 0a00 0000 0000 0000 ................
112	112	```
113	113
114		Presumably this is some sort of timestamp, but that's just a guess. Suggestions for ways of fixing this would be most welcome! (Otherwise, we'll just wait for Wheezy to go EOL and forget this ever happened. :trollface:)
	114	Presumably this is some sort of timestamp, but that's just a guess. Suggestions for ways of fixing this would be most welcome! (Otherwise, we'll just wait for Wheezy to go EOL and forget this ever happened. :trollface:) See [debuerreotype/debuerreotype#7](https://github.com/debuerreotype/debuerreotype/issues/7) for the current discussion of this issue.
115	115
116	116	## How much have you verified this?
117	117

-1

VERSION less more

0		0.1
	0	0.2

+23

-27

build-all.sh less more

1	1	set -Eeuo pipefail
2	2
3	3	suites=(
	4	unstable
	5	testing
	6	stable
4	7	oldstable
5		stable
6		testing
7		unstable
	8	oldoldstable
8	9
9		wheezy
10		jessie
11		stretch
12		sid
	10	# just in case (will no-op with "not supported on 'arch'" unless it exists)
	11	oldoldoldstable
13	12	)
14	13
15	14	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"

22	21	EOU
23	22	}
24	23	eusage() {
25		echo >&2 "error: $1"
	24	if [ "$#" -gt 0 ]; then
	25	echo >&2 "error: $*"
	26	fi
26	27	usage >&2
27	28	exit 1
28	29	}
29	30
30		# a silly flag to skip "docker build" (for giggles/debugging)
	31	options="$(getopt -n "$self" -o '' --long 'no-build' -- "$@")" \|\| eusage
	32	eval "set -- $options"
31	33	build=1
32		if [ "${1:-}" = '--no-build' ]; then
33		shift
34		build=
35		fi
	34	while true; do
	35	flag="$1"; shift
	36	case "$flag" in
	37	--no-build) build= ;; # for skipping "docker build"
	38	--) break ;;
	39	esac
	40	done
36	41
37	42	outputDir="${1:-}"; shift \|\| eusage 'missing output-dir'
38	43	timestamp="${1:-}"; shift \|\| eusage 'missing timestamp'

40	45	mkdir -p "$outputDir"
41	46	outputDir="$(readlink -f "$outputDir")"
42	47
43		dockerImage='tianon/debuerreotype'
	48	ver="$("$thisDir/scripts/debuerreotype-version")"
	49	ver="${ver%% *}"
	50	dockerImage="debuerreotype/debuerreotype:$ver"
44	51	[ -z "$build" ] \|\| docker build -t "$dockerImage" "$thisDir"
45	52
46	53	mirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp")"

51	58	echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --"
52	59	echo
53	60
54		fetch_codename() {
55		local suite="$1"; shift
56		wget -qO- "$mirror/dists/$suite/Release" \
57		\| tac\|tac \
58		\| awk -F ': ' 'tolower($1) == "codename" { print $2; exit }'
59		}
60		declare -A codenames=(
61		[testing]="$(fetch_codename 'testing')"
62		[unstable]="$(fetch_codename 'unstable')"
63		)
64
65	61	for suite in "${suites[@]}"; do
66	62	testUrl="$secmirror/dists/$suite/updates/main/binary-$dpkgArch/Packages.gz"
67	63	case "$suite" in
68		testing\|unstable\|"${codenames[testing]}"\|"${codenames[unstable]}")
	64	testing\|unstable)
69	65	testUrl="$mirror/dists/$suite/main/binary-$dpkgArch/Packages.gz"
70	66	;;
71	67	esac

75	71	echo >&2
76	72	continue
77	73	fi
78		"$thisDir/build.sh" --no-build "$outputDir" "$suite" "$timestamp"
	74	"$thisDir/build.sh" --no-build --codename-copy "$outputDir" "$suite" "$timestamp"
79	75	done

+138

-22

build.sh less more

7	7	cat <<-EOU
8	8	usage: $self <output-dir> <suite> <timestamp>
9	9	ie: $self output stretch 2017-05-08T00:00:00Z
	10	$self --codename-copy output stable 2017-05-08T00:00:00Z
10	11	EOU
11	12	}
12	13	eusage() {
13		echo >&2 "error: $1"
	14	if [ "$#" -gt 0 ]; then
	15	echo >&2 "error: $*"
	16	fi
14	17	usage >&2
15	18	exit 1
16	19	}
17	20
18		# a silly flag to skip "docker build" (for "build-all.sh")
	21	options="$(getopt -n "$self" -o '' --long 'no-build,codename-copy' -- "$@")" \|\| eusage
	22	eval "set -- $options"
19	23	build=1
20		if [ "${1:-}" = '--no-build' ]; then
21		shift
22		build=
23		fi
	24	codenameCopy=
	25	while true; do
	26	flag="$1"; shift
	27	case "$flag" in
	28	--no-build) build= ;; # for skipping "docker build"
	29	--codename-copy) codenameCopy=1 ;; # for copying a "stable.tar.xz" to "stretch.tar.xz" with updated sources.list (saves a lot of extra building work)
	30	--) break ;;
	31	esac
	32	done
24	33
25	34	outputDir="${1:-}"; shift \|\| eusage 'missing output-dir'
26	35	suite="${1:-}"; shift \|\| eusage 'missing suite'

39	48	)
40	49	fi
41	50
42		dockerImage='tianon/debuerreotype'
	51	ver="$("$thisDir/scripts/debuerreotype-version")"
	52	ver="${ver%% *}"
	53	dockerImage="debuerreotype/debuerreotype:$ver"
43	54	[ -z "$build" ] \|\| docker build -t "$dockerImage" "$thisDir"
44	55
45	56	docker run \

49	60	-w /tmp \
50	61	-e suite="$suite" \
51	62	-e timestamp="$timestamp" \
	63	-e codenameCopy="$codenameCopy" \
52	64	-e TZ='UTC' -e LC_ALL='C' \
53	65	"$dockerImage" \
54	66	bash -Eeuo pipefail -c '

56	68
57	69	epoch="$(date --date "$timestamp" +%s)"
58	70	serial="$(date --date "@$epoch" +%Y%m%d)"
	71	dpkgArch="$(dpkg --print-architecture)"
	72
59	73	exportDir="output"
60		outputDir="$exportDir/$serial"
61		dpkgArch="$(dpkg --print-architecture)"
	74	outputDir="$exportDir/$serial/$dpkgArch/$suite"
	75
	76	touch_epoch() {
	77	while [ "$#" -gt 0 ]; do
	78	local f="$1"; shift
	79	touch --no-dereference --date="@$epoch" "$f"
	80	done
	81	}
	82
	83	debuerreotypeScriptsDir="$(dirname "$(readlink -f "$(which debuerreotype-init)")")"
	84
	85	for archive in "" security; do
	86	snapshotUrl="$("$debuerreotypeScriptsDir/.snapshot-url.sh" "@$epoch" "${archive:+debian-${archive}}")"
	87	snapshotUrlFile="$exportDir/$serial/$dpkgArch/snapshot-url${archive:+-${archive}}"
	88	mkdir -p "$(dirname "$snapshotUrlFile")"
	89	echo "$snapshotUrl" > "$snapshotUrlFile"
	90	touch_epoch "$snapshotUrlFile"
	91	done
	92
	93	snapshotUrl="$(< "$exportDir/$serial/$dpkgArch/snapshot-url")"
	94	mkdir -p "$outputDir"
	95	wget -O "$outputDir/Release.gpg" "$snapshotUrl/dists/$suite/Release.gpg"
	96	wget -O "$outputDir/Release" "$snapshotUrl/dists/$suite/Release"
	97	gpgv \
	98	--keyring /usr/share/keyrings/debian-archive-keyring.gpg \
	99	--keyring /usr/share/keyrings/debian-archive-removed-keys.gpg \
	100	"$outputDir/Release.gpg" \
	101	"$outputDir/Release"
	102
	103	codename="$(awk -F ": " "\$1 == \"Codename\" { print \$2; exit }" "$outputDir/Release")"
	104	if [ -n "$codenameCopy" ] && [ "$codename" = "$suite" ]; then
	105	# if codename already is the same as suite, then making a copy does not make any sense
	106	codenameCopy=
	107	fi
	108	if [ -n "$codenameCopy" ] && [ -z "$codename" ]; then
	109	echo >&2 "error: --codename-copy specified but we failed to get a Codename for $suite"
	110	exit 1
	111	fi
62	112
63	113	{
64	114	debuerreotype-init rootfs "$suite" "@$epoch"

67	117	debuerreotype-apt-get rootfs update -qq
68	118	debuerreotype-apt-get rootfs dist-upgrade -yqq
69	119
70		# make a copy of rootfs so we can have a "slim" output too
71		mkdir -p rootfs-slim
72		tar -cC rootfs . \| tar -xC rootfs-slim
	120	# make a couple copies of rootfs so we can create other variants
	121	for variant in slim sbuild; do
	122	mkdir "rootfs-$variant"
	123	tar -cC rootfs . \| tar -xC "rootfs-$variant"
	124	done
73	125
74	126	# prefer iproute2 if it exists
75	127	iproute=iproute2

77	129	# poor wheezy
78	130	iproute=iproute
79	131	fi
80
81	132	debuerreotype-apt-get rootfs install -y --no-install-recommends inetutils-ping $iproute
82	133
83	134	debuerreotype-slimify rootfs-slim
84	135
85		du -hs rootfs rootfs-slim
	136	# this should match the list added to the "buildd" variant in debootstrap and the list installed by sbuild
	137	# https://anonscm.debian.org/cgit/d-i/debootstrap.git/tree/scripts/sid?id=706a45681c5bba5e062a9b02e19f079cacf2a3e8#n26
	138	# https://anonscm.debian.org/cgit/buildd-tools/sbuild.git/tree/bin/sbuild-createchroot?id=eace3d3e59e48d26eaf069d9b63a6a4c868640e6#n194
	139	debuerreotype-apt-get rootfs-sbuild install -y --no-install-recommends build-essential fakeroot
	140
	141	create_artifacts() {
	142	local targetBase="$1"; shift
	143	local rootfs="$1"; shift
	144	local suite="$1"; shift
	145	local variant="$1"; shift
	146
	147	# make a copy of the snapshot-facing sources.list file before we overwrite it
	148	cp "$rootfs/etc/apt/sources.list" "$targetBase.sources-list-snapshot"
	149	touch_epoch "$targetBase.sources-list-snapshot"
	150
	151	if [ "$variant" != "sbuild" ]; then
	152	debuerreotype-gen-sources-list "$rootfs" "$suite" http://deb.debian.org/debian http://security.debian.org
	153	debuerreotype-tar "$rootfs" "$targetBase.tar.xz"
	154	else
	155	# sbuild needs "deb-src" entries
	156	debuerreotype-gen-sources-list --deb-src "$rootfs" "$suite" http://deb.debian.org/debian http://security.debian.org
	157
	158	# schroot is picky about "/dev" (which is excluded by default in "debuerreotype-tar")
	159	# see https://github.com/debuerreotype/debuerreotype/pull/8#issuecomment-305855521
	160	debuerreotype-tar --include-dev "$rootfs" "$targetBase.tar.xz"
	161	fi
	162	du -hsx "$targetBase.tar.xz"
	163
	164	sha256sum "$targetBase.tar.xz" \| cut -d" " -f1 > "$targetBase.tar.xz.sha256"
	165	touch_epoch "$targetBase.tar.xz.sha256"
	166
	167	debuerreotype-chroot "$rootfs" dpkg-query -W > "$targetBase.manifest"
	168	echo "$epoch" > "$targetBase.debuerreotype-epoch"
	169	touch_epoch "$targetBase.manifest" "$targetBase.debuerreotype-epoch"
	170
	171	for f in debian_version os-release apt/sources.list; do
	172	targetFile="$targetBase.$(basename "$f" \| sed -r "s/[^a-zA-Z0-9_-]+/-/g")"
	173	cp "$rootfs/etc/$f" "$targetFile"
	174	touch_epoch "$targetFile"
	175	done
	176	}
86	177
87	178	for rootfs in rootfs*/; do
88		debuerreotype-gen-sources-list "$rootfs" "$suite" http://deb.debian.org/debian http://security.debian.org
	179	rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ...
	180
	181	du -hsx "$rootfs"
	182
	183	variant="${rootfs#rootfs}" # "", "-slim", ...
	184	variant="${variant#-}" # "", "slim", ...
	185
	186	variantDir="$outputDir/$variant"
	187	mkdir -p "$variantDir"
	188
	189	targetBase="$variantDir/rootfs"
	190
	191	create_artifacts "$targetBase" "$rootfs" "$suite" "$variant"
89	192	done
90	193
91		mkdir -p "$outputDir"
92		for variant in "" -slim; do
93		targetBase="$outputDir/$suite$variant-$dpkgArch"
94		debuerreotype-tar "rootfs$variant" "$targetBase.tar.xz"
95		debuerreotype-chroot "rootfs$variant" dpkg-query -W > "$targetBase.manifest"
96		touch --no-dereference --date="@$epoch" "$targetBase.manifest"
97		done
	194	if [ -n "$codenameCopy" ]; then
	195	codenameDir="$exportDir/$serial/$dpkgArch/$codename"
	196	mkdir -p "$codenameDir"
	197	tar -cC "$outputDir" --exclude="*/rootfs." . \| tar -xC "$codenameDir"
	198
	199	for rootfs in rootfs*/; do
	200	rootfs="${rootfs%/}" # "rootfs", "rootfs-slim", ...
	201
	202	variant="${rootfs#rootfs}" # "", "-slim", ...
	203	variant="${variant#-}" # "", "slim", ...
	204
	205	variantDir="$codenameDir/$variant"
	206	targetBase="$variantDir/rootfs"
	207
	208	# point sources.list back at snapshot.debian.org temporarily (but this time pointing at $codename instead of $suite)
	209	debuerreotype-gen-sources-list "$rootfs" "$codename" "$(< "$exportDir/$serial/$dpkgArch/snapshot-url")" "$(< "$exportDir/$serial/$dpkgArch/snapshot-url-security")"
	210
	211	create_artifacts "$targetBase" "$rootfs" "$codename" "$variant"
	212	done
	213	fi
98	214	} >&2
99	215
100	216	tar -cC "$exportDir" .

-1

scripts/.constants.sh less more

31	31	EOU
32	32	}
33	33	eusage() {
34		echo >&2 "error: $1"
	34	if [ "$#" -gt 0 ]; then
	35	echo >&2 "error: $*"
	36	fi
35	37	usage >&2
36	38	exit 1
37	39	}

-1

scripts/.tar-exclude less more

0	0	# the file we store the "epoch" of a given rootfs in
1	1	./debuerreotype-epoch
2	2
3		./dev/**
	3	# "/dev" is a special case in "debuerreotype-tar"
	4	#./dev/**
4	5	./proc/**
5	6	./sys/**
6	7

+23

-5

scripts/debuerreotype-gen-sources-list less more

2	2
3	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
4	4	source "$thisDir/.constants.sh" \
5		'<target-dir> <suite> <mirror> <secmirror>' \
	5	'[--deb-src] <target-dir> <suite> <mirror> <secmirror>' \
6	6	'rootfs stretch http://deb.debian.org/debian http://security.debian.org'
	7
	8	options="$(getopt -n "$self" -o '' --long 'deb-src' -- "$@")" \|\| eusage
	9	eval "set -- $options"
	10	debSrc=
	11	while true; do
	12	flag="$1"; shift
	13	case "$flag" in
	14	--deb-src) debSrc=1 ;;
	15	--) break ;;
	16	esac
	17	done
7	18
8	19	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
9	20	suite="${1:-}"; shift \|\| eusage 'missing suite'

13	24
14	25	comp='main'
15	26
	27	deb() {
	28	echo "deb $*"
	29	if [ -n "$debSrc" ]; then
	30	echo "deb-src $*"
	31	fi
	32	}
	33
16	34	# https://github.com/tianon/go-aptsources/blob/e066ed9cd8cd9eef7198765bd00ec99679e6d0be/target.go#L16-L58
17	35	{
18	36	case "$suite" in
19	37	sid\|unstable\|testing)
20		echo "deb $mirror $suite $comp"
	38	deb "$mirror" "$suite" "$comp"
21	39	;;
22	40
23	41	*)
24		echo "deb $mirror $suite $comp"
25		echo "deb $mirror $suite-updates $comp"
26		echo "deb $secmirror $suite/updates $comp"
	42	deb "$mirror" "$suite" "$comp"
	43	deb "$mirror" "$suite-updates" "$comp"
	44	deb "$secmirror" "$suite/updates" "$comp"
27	45	;;
28	46	esac
29	47	} > "$targetDir/etc/apt/sources.list"

+31

-9

scripts/debuerreotype-tar less more

2	2
3	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
4	4	source "$thisDir/.constants.sh" \
5		'<target-dir> <target-tar>' \
	5	'[--include-dev] <target-dir> <target-tar>' \
6	6	'rootfs rootfs.tar'
	7
	8	options="$(getopt -n "$self" -o '' --long 'include-dev' -- "$@")" \|\| eusage
	9	eval "set -- $options"
	10	includeDev=
	11	while true; do
	12	flag="$1"; shift
	13	case "$flag" in
	14	--include-dev) includeDev=1 ;;
	15	--) break ;;
	16	esac
	17	done
7	18
8	19	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
9	20	[ -n "$targetDir" ]

14	25	[ -n "$epoch" ]
15	26
16	27	"$thisDir/debuerreotype-fixup" "$targetDir"
17		tar --create \
18		--file "$targetTar" \
19		--auto-compress \
20		--directory "$targetDir" \
21		--exclude-from "$thisDir/.tar-exclude" \
22		--numeric-owner \
23		--transform 's,^./,,' \
24		--sort name \
	28
	29	tarArgs=(
	30	--create
	31	--file "$targetTar"
	32	--auto-compress
	33	--directory "$targetDir"
	34	--exclude-from "$thisDir/.tar-exclude"
	35	)
	36	if [ -z "$includeDev" ]; then
	37	tarArgs+=( --exclude './dev/**' )
	38	fi
	39	tarArgs+=(
	40	--numeric-owner
	41	--transform 's,^./,,'
	42	--sort name
25	43	.
	44	)
	45
	46	tar "${tarArgs[@]}"
	47
26	48	touch --no-dereference --date="@$epoch" "$targetTar"