Commit 2e429bc5783f8bf2f1cfb2954c1c1a90afeb902e - debuerreotype

+2

-0

.dockerignore less more

	0	**
	1	!scripts/

+13

-0

.travis.sh less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	epoch="$(TZ=UTC date --date "$TIMESTAMP" +%s)"
	4	serial="$(TZ=UTC date --date "@$epoch" +%Y%m%d)"
	5
	6	set -x
	7
	8	./scripts/debuerreotype-version
	9	./build.sh travis "$SUITE" "@$epoch"
	10
	11	real="$(sha256sum "travis/$serial/$SUITE-amd64.tar.xz" \| cut -d' ' -f1)"
	12	[ -z "$SHA256" ] \|\| [ "$SHA256" = "$real" ]

+13

-0

.travis.yml less more

	0	language: bash
	1	services: docker
	2
	3	env:
	4	- SUITE=jessie TIMESTAMP=2017-01-01T00:00:00Z SHA256=a674379d30cf457e49a909bb0e254ddcb1f72c27f45c68fe930668d6d9399232
	5	- SUITE=stretch TIMESTAMP=2017-01-01T00:00:00Z SHA256=139ed970d52ef950c223f9ab325657eb93d0a93c7d6e2fc697fe7510e61760fa
	6	- SUITE=sid TIMESTAMP=2017-01-01T00:00:00Z SHA256=b75b4496deb4d6cee32245e4125e7ef948b09afbeb1ef3b9669e56daf3e822a7
	7
	8	script:
	9	- travis_retry ./.travis.sh
	10
	11	after_script:
	12	- docker images

+38

-0

Dockerfile less more

	0	# docker run --cap-add SYS_ADMIN --tmpfs /tmp:dev,exec,suid,noatime ...
	1
	2	# bootstrapping a new architecture?
	3	# ./scripts/debuerreotype-init /tmp/docker-rootfs stretch now
	4	# ./scripts/debuerreotype-minimizing-config /tmp/docker-rootfs
	5	# ./scripts/debuerreotype-gen-sources-list /tmp/docker-rootfs stretch http://deb.debian.org/debian http://security.debian.org
	6	# ./scripts/debuerreotype-tar /tmp/docker-rootfs - \| docker import - debian:stretch-slim
	7	# alternate:
	8	# debootstrap --variant=minbase stretch /tmp/docker-rootfs
	9	# tar -cC /tmp/docker-rootfs . \| docker import - debian:stretch-slim
	10	# (or your own favorite set of "debootstrap" commands to create a base image for building this one FROM)
	11	FROM debian:stretch-slim
	12
	13	RUN apt-get update && apt-get install -y --no-install-recommends \
	14	debootstrap \
	15	xz-utils \
	16	&& rm -rf /var/lib/apt/lists/*
	17
	18	COPY scripts /opt/debuerreotype/scripts
	19	RUN set -ex; \
	20	cd /opt/debuerreotype/scripts; \
	21	for f in debuerreotype-*; do \
	22	ln -svL "$PWD/$f" "/usr/local/bin/$f"; \
	23	done
	24
	25	WORKDIR /tmp
	26
	27	# a few example md5sum values for amd64:
	28
	29	# debuerreotype-init test-stretch stretch 2017-05-08T00:00:00Z
	30	# debuerreotype-tar test-stretch test-stretch.tar
	31	# md5sum test-stretch.tar
	32	# 6f965e84837215ac0aa375e3391392db
	33
	34	# debuerreotype-init test-jessie jessie 2017-05-08T00:00:00Z
	35	# debuerreotype-tar test-jessie test-jessie.tar
	36	# md5sum test-jessie.tar
	37	# 45624a45af50f60b6b4be7203bf16c86

+7

-0

LICENSE less more

	0	Copyright 2017 Tianon Gravi <tianon@debian.org>
	1
	2	Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
	3
	4	The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
	5
	6	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

+123

-0

README.md less more

	0	# Debuerreotype
	1
	2	[![Build Status](https://travis-ci.org/debuerreotype/debuerreotype.svg?branch=master)](https://travis-ci.org/debuerreotype/debuerreotype/branches)
	3
	4	Reproducible, [snapshot](http://snapshot.debian.org)-based Debian rootfs builds (especially for Docker).
	5
	6	This is based on [lamby](https://github.com/lamby)'s work for reproducible `debootstrap`:
	7
	8	- https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d
	9	- https://wiki.debian.org/ReproducibleInstalls
	10
	11	### "Debuerreotype"?
	12
	13	The name is an attempt at riffing off the photography basis of the word "snapshot". The [daguerreotype](https://en.wikipedia.org/wiki/Daguerreotype) process was an early method for taking photographs, and this is a method for taking "photographs" of Debian at a given point in time.
	14
	15	## Why?
	16
	17	The goal is to create an auditable, reproducible process for creating rootfs tarballs (especially for use in Docker) of Debian releases, based on point-in-time snapshots from [snapshot.debian.org](http://snapshot.debian.org).
	18
	19	However, as noted below, the only strictly Docker-specific script is `debuerreotype-minimizing-config`, which applies many configuration tweaks which are useful for Docker users and may or may not be useful outside of that context.
	20
	21	## Usage
	22
	23	The usage of the scripts here center around a "rootfs" directory, which is both the working directory for building the target rootfs, and contains the `debuerreotype-epoch` file, which records our snapshot.debian.org epoch value (so we can adjust timestamps using it, as it is the basis for our reproducibility).
	24
	25	Available scripts:
	26
	27	\| script \| purpose \|
	28	\| --- \| --- \|
	29	\| `debuerreotype-init` \| create the initial "rootfs", given a suite and a timestamp (in some format `date(1)` can parse); `sources.list` will be pointing at snapshot.debian.org \|
	30	\| `debuerreotype-chroot` \| run a command in the given "rootfs" (using `unshare` to mount `/dev`, `/proc`, and `/sys` from the parent environment in a simple, safe way) \|
	31	\| `debuerreotype-apt-get` \| run `apt-get` via `debuerreotype-chroot`, including `-o Acquire::Check-Valid-Until=false` to account for older snapshots with (now) invalid `Valid-Until` values \|
	32	\| `debuerreotype-minimizing-config` \| apply configuration tweaks to make the rootfs minimal and keep it minimal (especially targeted at Docker images, with comments explicitly describing Docker use cases) \|
	33	\| `debuerreotype-slimify` \| remove files such as documentation to create an even smaller rootfs (used for creating `slim` variants of the Docker images, for example) \|
	34	\| `debuerreotype-gen-sources-list` \| generate an appropriate `sources.list` in the rootfs given a suite, mirror, and secmirror (especially for updating `sources.list` to point at deb.debian.org before generating outputs) \|
	35	\| `debuerreotype-fixup` \| invoked by `debuerreotype-tar` to fixup timestamps and remove known-bad log files for determinism \|
	36	\| `debuerreotype-tar` \| deterministically create a tar file of the rootfs \|
	37	\| `debuerreotype-version` \| print out the version of the current `debuerreotype` installation \|
	38
	39	A simple `Dockerfile` is provided for using these scripts in a simple deterministic environment based on Docker, but given a recent enough version of `debootstrap`, they should run fine outside Docker as well (and their deterministic properties have been verified on at least a Gentoo host in addition to the provided Debian-based Docker environment).
	40
	41	The provided `Dockerfile` also includes comments with hints for bootstrapping the environment on a new architecture (which then presumably doesn't have a `debian` Docker base image yet).
	42
	43	Full example: (see [`build.sh`](build.sh) for this in practice)
	44
	45	```console
	46	$ debuerreotype-init rootfs stretch 2017-01-01T00:00:00Z
	47	I: Retrieving InRelease
	48	I: Checking Release signature
	49	I: Valid Release signature (key id 126C0D24BD8A2942CC7DF8AC7638D0442B90D010)
	50	...
	51	I: Checking component main on http://snapshot.debian.org/archive/debian/20170101T000000Z...
	52	...
	53	I: Base system installed successfully.
	54
	55	$ cat rootfs/debuerreotype-epoch
	56	1483228800
	57
	58	$ debuerreotype-minimizing-config rootfs
	59
	60	$ debuerreotype-apt-get rootfs update -qq
	61	$ debuerreotype-apt-get rootfs dist-upgrade -yqq
	62	$ debuerreotype-apt-get rootfs install -yqq --no-install-recommends inetutils-ping iproute2
	63	debconf: delaying package configuration, since apt-utils is not installed
	64	Selecting previously unselected package libelf1:amd64.
	65	(Reading database ... 6299 files and directories currently installed.)
	66	Preparing to unpack .../0-libelf1_0.166-2.2_amd64.deb ...
	67	Unpacking libelf1:amd64 (0.166-2.2) ...
	68	Selecting previously unselected package libmnl0:amd64.
	69	Preparing to unpack .../1-libmnl0_1.0.4-2_amd64.deb ...
	70	Unpacking libmnl0:amd64 (1.0.4-2) ...
	71	Selecting previously unselected package iproute2.
	72	Preparing to unpack .../2-iproute2_4.9.0-1_amd64.deb ...
	73	Unpacking iproute2 (4.9.0-1) ...
	74	Selecting previously unselected package netbase.
	75	Preparing to unpack .../3-netbase_5.3_all.deb ...
	76	Unpacking netbase (5.3) ...
	77	Selecting previously unselected package inetutils-ping.
	78	Preparing to unpack .../4-inetutils-ping_2%3a1.9.4-2+b1_amd64.deb ...
	79	Unpacking inetutils-ping (2:1.9.4-2+b1) ...
	80	Setting up libelf1:amd64 (0.166-2.2) ...
	81	Processing triggers for libc-bin (2.24-8) ...
	82	Setting up libmnl0:amd64 (1.0.4-2) ...
	83	Setting up netbase (5.3) ...
	84	Setting up inetutils-ping (2:1.9.4-2+b1) ...
	85	Setting up iproute2 (4.9.0-1) ...
	86	Processing triggers for libc-bin (2.24-8) ...
	87
	88	$ debuerreotype-gen-sources-list rootfs stretch http://deb.debian.org/debian http://security.debian.org
	89
	90	$ debuerreotype-tar rootfs - \| sha256sum
	91	0542bec04135ed60ed5763f0bcf90381d4e5e33786d57aba5aa4b0fc4e43478a -
	92
	93	$ # try it! you should get that same sha256sum value!
	94	```
	95
	96	## Why isn't Wheezy reproducible??
	97
	98	Wheezy is a little sad, and will have a delta similar to the following (as seen via [`diffoscope`](https://diffoscope.org/)):
	99
	100	```
	101	├── etc/apt/trustdb.gpg
	102	│ │ @@ -1,8 +1,8 @@
	103	│ │ -0000000: 0167 7067 0303 0105 0102 0000 591b faa5 .gpg........Y...
	104	│ │ +0000000: 0167 7067 0303 0105 0102 0000 591b fc0c .gpg........Y...
	105	│ │ 0000010: 0000 0000 0000 0000 0000 0000 0000 0000 ................
	106	│ │ 0000020: 0000 0000 0000 0001 0a00 0000 0000 0000 ................
	107	│ │ 0000030: 0000 0000 0000 0000 0000 0000 0000 0000 ................
	108	│ │ 0000040: 0000 0000 0000 0000 0000 0000 0000 0000 ................
	109	│ │ 0000050: 0a00 0000 0000 0000 0000 0000 0000 0000 ................
	110	│ │ 0000060: 0000 0000 0000 0000 0000 0000 0000 0000 ................
	111	│ │ 0000070: 0000 0000 0000 0000 0a00 0000 0000 0000 ................
	112	```
	113
	114	Presumably this is some sort of timestamp, but that's just a guess. Suggestions for ways of fixing this would be most welcome! (Otherwise, we'll just wait for Wheezy to go EOL and forget this ever happened. :trollface:)
	115
	116	## How much have you verified this?
	117
	118	Well, I ran the scripts across seven explicit architectures (`amd64`, `arm64`, `armel`, `armhf`, `i386`, `ppc64el`, `s390x`) and eight explicit suites (`oldstable`, `stable`, `testing`, `unstable`, `wheezy`, `jessie`, `stretch`, `sid`) for a timestamp of `2017-05-16T00:00:00Z` (where supported, since `wheezy`/`oldstable` didn't or no longer currently supports some of those architectures), and the above `wheezy` delta (a few bytes in `etc/apt/trustdb.gpg`) were the _only_ modification to any of the tarballs after several runs across several days.
	119
	120	Additionally, Travis runs with a fixed timestamp value across several suites to verify that their checksums are reproducible, as expected.
	121
	122	From time to time, comments in the files generated by `debuerreotype-minimizing-config` might change (for example), which would obviously result in a different checksum, but a simple [`diffoscope`](https://diffoscope.org/) should be sufficient to verify that the change is benign.

+1

-0

VERSION less more

0

0.1

+80

-0

build-all.sh less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	suites=(
	4	oldstable
	5	stable
	6	testing
	7	unstable
	8
	9	wheezy
	10	jessie
	11	stretch
	12	sid
	13	)
	14
	15	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	16	self="$(basename "$0")"
	17
	18	usage() {
	19	cat <<-EOU
	20	usage: $self <output-dir> <timestamp>
	21	ie: $self output 2017-05-08T00:00:00Z
	22	EOU
	23	}
	24	eusage() {
	25	echo >&2 "error: $1"
	26	usage >&2
	27	exit 1
	28	}
	29
	30	# a silly flag to skip "docker build" (for giggles/debugging)
	31	build=1
	32	if [ "${1:-}" = '--no-build' ]; then
	33	shift
	34	build=
	35	fi
	36
	37	outputDir="${1:-}"; shift \|\| eusage 'missing output-dir'
	38	timestamp="${1:-}"; shift \|\| eusage 'missing timestamp'
	39
	40	mkdir -p "$outputDir"
	41	outputDir="$(readlink -f "$outputDir")"
	42
	43	dockerImage='tianon/debuerreotype'
	44	[ -z "$build" ] \|\| docker build -t "$dockerImage" "$thisDir"
	45
	46	mirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp")"
	47	secmirror="$("$thisDir/scripts/.snapshot-url.sh" "$timestamp" 'debian-security')"
	48
	49	dpkgArch="$(docker run --rm "$dockerImage" dpkg --print-architecture)"
	50	echo
	51	echo "-- BUILDING TARBALLS FOR '$dpkgArch' FROM '$mirror/' --"
	52	echo
	53
	54	fetch_codename() {
	55	local suite="$1"; shift
	56	wget -qO- "$mirror/dists/$suite/Release" \
	57	\| tac\|tac \
	58	\| awk -F ': ' 'tolower($1) == "codename" { print $2; exit }'
	59	}
	60	declare -A codenames=(
	61	[testing]="$(fetch_codename 'testing')"
	62	[unstable]="$(fetch_codename 'unstable')"
	63	)
	64
	65	for suite in "${suites[@]}"; do
	66	testUrl="$secmirror/dists/$suite/updates/main/binary-$dpkgArch/Packages.gz"
	67	case "$suite" in
	68	testing\|unstable\|"${codenames[testing]}"\|"${codenames[unstable]}")
	69	testUrl="$mirror/dists/$suite/main/binary-$dpkgArch/Packages.gz"
	70	;;
	71	esac
	72	if ! wget --quiet --spider "$testUrl"; then
	73	echo >&2
	74	echo >&2 "warning: '$suite' not supported on '$dpkgArch' (at '$timestamp'); skipping"
	75	echo >&2
	76	continue
	77	fi
	78	"$thisDir/build.sh" --no-build "$outputDir" "$suite" "$timestamp"
	79	done

+102

-0

build.sh less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	self="$(basename "$0")"
	5
	6	usage() {
	7	cat <<-EOU
	8	usage: $self <output-dir> <suite> <timestamp>
	9	ie: $self output stretch 2017-05-08T00:00:00Z
	10	EOU
	11	}
	12	eusage() {
	13	echo >&2 "error: $1"
	14	usage >&2
	15	exit 1
	16	}
	17
	18	# a silly flag to skip "docker build" (for "build-all.sh")
	19	build=1
	20	if [ "${1:-}" = '--no-build' ]; then
	21	shift
	22	build=
	23	fi
	24
	25	outputDir="${1:-}"; shift \|\| eusage 'missing output-dir'
	26	suite="${1:-}"; shift \|\| eusage 'missing suite'
	27	timestamp="${1:-}"; shift \|\| eusage 'missing timestamp'
	28
	29	mkdir -p "$outputDir"
	30	outputDir="$(readlink -f "$outputDir")"
	31
	32	securityArgs=(
	33	--cap-add SYS_ADMIN
	34	)
	35	if docker info \| grep -q apparmor; then
	36	# AppArmor blocks mount :)
	37	securityArgs+=(
	38	--security-opt apparmor=unconfined
	39	)
	40	fi
	41
	42	dockerImage='tianon/debuerreotype'
	43	[ -z "$build" ] \|\| docker build -t "$dockerImage" "$thisDir"
	44
	45	docker run \
	46	--rm \
	47	"${securityArgs[@]}" \
	48	--tmpfs /tmp:dev,exec,suid,noatime \
	49	-w /tmp \
	50	-e suite="$suite" \
	51	-e timestamp="$timestamp" \
	52	-e TZ='UTC' -e LC_ALL='C' \
	53	"$dockerImage" \
	54	bash -Eeuo pipefail -c '
	55	set -x
	56
	57	epoch="$(date --date "$timestamp" +%s)"
	58	serial="$(date --date "@$epoch" +%Y%m%d)"
	59	exportDir="output"
	60	outputDir="$exportDir/$serial"
	61	dpkgArch="$(dpkg --print-architecture)"
	62
	63	{
	64	debuerreotype-init rootfs "$suite" "@$epoch"
	65
	66	debuerreotype-minimizing-config rootfs
	67	debuerreotype-apt-get rootfs update -qq
	68	debuerreotype-apt-get rootfs dist-upgrade -yqq
	69
	70	# make a copy of rootfs so we can have a "slim" output too
	71	mkdir -p rootfs-slim
	72	tar -cC rootfs . \| tar -xC rootfs-slim
	73
	74	# prefer iproute2 if it exists
	75	iproute=iproute2
	76	if ! debuerreotype-chroot rootfs apt-cache show iproute2 > /dev/null; then
	77	# poor wheezy
	78	iproute=iproute
	79	fi
	80
	81	debuerreotype-apt-get rootfs install -y --no-install-recommends inetutils-ping $iproute
	82
	83	debuerreotype-slimify rootfs-slim
	84
	85	du -hs rootfs rootfs-slim
	86
	87	for rootfs in rootfs*/; do
	88	debuerreotype-gen-sources-list "$rootfs" "$suite" http://deb.debian.org/debian http://security.debian.org
	89	done
	90
	91	mkdir -p "$outputDir"
	92	for variant in "" -slim; do
	93	targetBase="$outputDir/$suite$variant-$dpkgArch"
	94	debuerreotype-tar "rootfs$variant" "$targetBase.tar.xz"
	95	debuerreotype-chroot "rootfs$variant" dpkg-query -W > "$targetBase.manifest"
	96	touch --no-dereference --date="@$epoch" "$targetBase.manifest"
	97	done
	98	} >&2
	99
	100	tar -cC "$exportDir" .
	101	' \| tar -xvC "$outputDir"

+38

-0

scripts/.constants.sh less more

	0	#!/usr/bin/env bash
	1
	2	# constants of the universe
	3	export TZ='UTC' LC_ALL='C'
	4	umask 0002
	5	scriptsDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	6
	7	_version() {
	8	local v
	9	if [ -r "$scriptsDir/../VERSION" ]; then
	10	v="$(< "$scriptsDir/../VERSION")"
	11	else
	12	v='unknown'
	13	fi
	14	if [ -d "$scriptsDir/../.git" ] && command -v git > /dev/null; then
	15	local commit="$(git -C "$scriptsDir" rev-parse --short 'HEAD^{commit}')"
	16	v="$v commit $commit"
	17	fi
	18	echo "$v"
	19	}
	20
	21	usageStr="$1"
	22	usageEx="$2"
	23	self="$(basename "$0")"
	24	usage() {
	25	local v="$(_version)"
	26	cat <<-EOU
	27	usage: $self $usageStr
	28	ie: $self $usageEx
	29
	30	debuerreotype version $v
	31	EOU
	32	}
	33	eusage() {
	34	echo >&2 "error: $1"
	35	usage >&2
	36	exit 1
	37	}

+10

-0

scripts/.slimify-excludes less more

	0	# This file contains the list of files/directories which will be removed for "slim" image variants.
	1	# https://github.com/tianon/docker-brew-debian/issues/48
	2	# https://wiki.ubuntu.com/ReducingDiskFootprint#Drop_unnecessary_files
	3	/usr/share/doc/*
	4	/usr/share/groff/*
	5	/usr/share/info/*
	6	/usr/share/linda/*
	7	/usr/share/lintian/*
	8	/usr/share/locale/*
	9	/usr/share/man/*

+13

-0

scripts/.snapshot-url.sh less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<timestamp> [archive]' \
	6	'2017-05-08T00:00:00Z debian-security'
	7
	8	timestamp="${1:-}"; shift \|\| eusage 'missing timestamp'
	9	archive="${1:-debian}"
	10
	11	t="$(date --date "$timestamp" '+%Y%m%dT%H%M%SZ')"
	12	echo "http://snapshot.debian.org/archive/$archive/$t"

+19

-0

scripts/.tar-exclude less more

	0	# the file we store the "epoch" of a given rootfs in
	1	./debuerreotype-epoch
	2
	3	./dev/**
	4	./proc/**
	5	./sys/**
	6
	7	./var/cache/apt/**
	8	./var/lib/apt/lists/**
	9
	10	# ends up with host-kernel info
	11	./etc/apt/apt.conf.d/01autoremove-kernels
	12
	13	# useful data in these, but includes timestamps too
	14	./var/log/apt/history.log
	15	./var/log/apt/term.log
	16
	17	# wheezy-only file which contains host-kernel info
	18	./run/motd.dynamic

+12

-0

scripts/debuerreotype-apt-get less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir> arguments' \
	6	'rootfs update'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	[ -n "$targetDir" ]
	10
	11	"$thisDir/debuerreotype-chroot" "$targetDir" apt-get -o Acquire::Check-Valid-Until=false "$@"

+22

-0

scripts/debuerreotype-chroot less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir> <command> [args...]' \
	6	'rootfs apt-get update'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	cmd="${1:-}"; shift \|\| eusage 'missing command'
	10	[ -n "$targetDir" ]
	11	epoch="$(< "$targetDir/debuerreotype-epoch")"
	12	[ -n "$epoch" ]
	13
	14	export targetDir epoch
	15	unshare --mount bash -Eeuo pipefail -c '
	16	[ -n "$targetDir" ] # just to be safe
	17	for dir in dev proc sys; do
	18	mount --rbind "/$dir" "$targetDir/$dir"
	19	done
	20	exec chroot "$targetDir" /usr/bin/env -i PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" TZ="$TZ" LC_ALL="$LC_ALL" SOURCE_DATE_EPOCH="$epoch" "$@"
	21	' -- "$cmd" "$@"

+23

-0

scripts/debuerreotype-fixup less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir>' \
	6	'rootfs'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	[ -n "$targetDir" ]
	10	epoch="$(< "$targetDir/debuerreotype-epoch")"
	11	[ -n "$epoch" ]
	12
	13	# https://github.com/lamby/debootstrap/commit/66b15380814aa62ca4b5807270ac57a3c8a0558d#diff-de4eef4ab836e5c6c9c1f820a2f624baR709
	14	rm -f \
	15	"$targetDir/var/log/dpkg.log" \
	16	"$targetDir/var/log/bootstrap.log" \
	17	"$targetDir/var/log/alternatives.log" \
	18	"$targetDir/var/cache/ldconfig/aux-cache"
	19
	20	find "$targetDir" \
	21	-newermt "@$epoch" \
	22	-exec touch --no-dereference --date="@$epoch" '{}' +

+31

-0

scripts/debuerreotype-gen-sources-list less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir> <suite> <mirror> <secmirror>' \
	6	'rootfs stretch http://deb.debian.org/debian http://security.debian.org'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	suite="${1:-}"; shift \|\| eusage 'missing suite'
	10	mirror="${1:-}"; shift \|\| eusage 'missing mirror'
	11	secmirror="${1:-}"; shift \|\| eusage 'missing secmirror'
	12	[ -n "$targetDir" ]
	13
	14	comp='main'
	15
	16	# https://github.com/tianon/go-aptsources/blob/e066ed9cd8cd9eef7198765bd00ec99679e6d0be/target.go#L16-L58
	17	{
	18	case "$suite" in
	19	sid\|unstable\|testing)
	20	echo "deb $mirror $suite $comp"
	21	;;
	22
	23	*)
	24	echo "deb $mirror $suite $comp"
	25	echo "deb $mirror $suite-updates $comp"
	26	echo "deb $secmirror $suite/updates $comp"
	27	;;
	28	esac
	29	} > "$targetDir/etc/apt/sources.list"
	30	chmod 0644 "$targetDir/etc/apt/sources.list"

+70

-0

scripts/debuerreotype-init less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir> <suite> <timestamp>' \
	6	'rootfs stretch 2017-05-08T00:00:00Z'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	suite="${1:-}"; shift \|\| eusage 'missing suite'
	10	timestamp="${1:-}"; shift \|\| eusage 'missing timestamp'
	11	[ -n "$targetDir" ] # must be non-empty
	12
	13	if [ -e "$targetDir" ] && [ -z "$(find "$targetDir" -maxdepth 0 -empty)" ]; then
	14	echo >&2 "error: '$targetDir' already exists (and isn't empty)!"
	15	exit 1
	16	fi
	17
	18	epoch="$(date --date "$timestamp" '+%s')"
	19	export SOURCE_DATE_EPOCH="$epoch"
	20
	21	mirror="$("$thisDir/.snapshot-url.sh" "@$epoch")"
	22	secmirror="$("$thisDir/.snapshot-url.sh" "@$epoch" 'debian-security')"
	23
	24	debootstrap \
	25	--force-check-gpg \
	26	--merged-usr \
	27	--variant=minbase \
	28	"$suite" "$targetDir" "$mirror"
	29	echo "$epoch" > "$targetDir/debuerreotype-epoch"
	30
	31	"$thisDir/debuerreotype-gen-sources-list" "$targetDir" "$suite" "$mirror" "$secmirror"
	32
	33	# since we're minbase, we know everything included is either essential, or a dependency of essential, so let's get clean "apt-mark showmanual" output
	34	"$thisDir/debuerreotype-chroot" "$targetDir" apt-mark auto '.*' > /dev/null
	35
	36	echo 'debuerreotype' > "$targetDir/etc/hostname"
	37	echo "$epoch" \
	38	\| md5sum \
	39	\| cut -f1 -d' ' \
	40	> "$targetDir/etc/machine-id" # TODO should we only do this if "/etc/machine-id" already exists?
	41	{
	42	echo 'nameserver 8.8.8.8'
	43	echo 'nameserver 8.8.4.4'
	44	} > "$targetDir/etc/resolv.conf"
	45	chmod 0644 \
	46	"$targetDir/etc/hostname" \
	47	"$targetDir/etc/machine-id" \
	48	"$targetDir/etc/resolv.conf"
	49
	50	# https://bugs.debian.org/857803
	51	# adjust field 3 in /etc/shadow and /etc/shadow- to $(( epoch / 60 / 60 / 24 )), if it's larger
	52	sp_lstchg="$(( epoch / 60 / 60 / 24 ))"
	53	for shadowFile in etc/shadow etc/shadow-; do
	54	newShadowFile="$shadowFile.debuerreotype"
	55	awk -F ':' \
	56	-v OFS=':' \
	57	-v sp_lstchg="$sp_lstchg" \
	58	'{
	59	if ($3 > sp_lstchg) {
	60	$3 = sp_lstchg
	61	}
	62	print
	63	}' "$targetDir/$shadowFile" > "$targetDir/$newShadowFile"
	64	if [ "$(< "$targetDir/$shadowFile")" != "$(< "$targetDir/$newShadowFile")" ]; then
	65	# use "cat" instead of "mv" so permissions don't change
	66	cat "$targetDir/$newShadowFile" > "$targetDir/$shadowFile"
	67	fi
	68	rm -f "$targetDir/$newShadowFile"
	69	done

+112

-0

scripts/debuerreotype-minimizing-config less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir>' \
	6	'rootfs'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	[ -n "$targetDir" ]
	10
	11	# https://github.com/docker/docker/blob/d6f4fe9e38b60f63e429fff7ffced9c26cbf8236/contrib/mkimage/debootstrap#L63-L177
	12
	13	# prevent init scripts from running during install/update
	14	cat > "$targetDir/usr/sbin/policy-rc.d" <<-'EOF'
	15	#!/bin/sh
	16
	17	# For most Docker users, "apt-get install" only happens during "docker build",
	18	# where starting services doesn't work and often fails in humorous ways. This
	19	# prevents those failures by stopping the services from attempting to start.
	20
	21	exit 101
	22	EOF
	23	chmod 0755 "$targetDir/usr/sbin/policy-rc.d"
	24
	25	# prevent upstart scripts from running during install/update
	26	"$thisDir/debuerreotype-chroot" "$targetDir" dpkg-divert --local --rename --add /sbin/initctl > /dev/null
	27	cp -a "$targetDir/usr/sbin/policy-rc.d" "$targetDir/sbin/initctl"
	28	sed -i 's/^exit.*/exit 0/' "$targetDir/sbin/initctl"
	29	# TODO should we only do this if "/sbin/initctl" already exists?
	30
	31	# force dpkg not to call sync() after package extraction (speeding up installs)
	32	cat > "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" <<-'EOF'
	33	# For most Docker users, package installs happen during "docker build", which
	34	# doesn't survive power loss and gets restarted clean afterwards anyhow, so
	35	# this minor tweak gives us a nice speedup (much nicer on spinning disks,
	36	# obviously).
	37
	38	force-unsafe-io
	39	EOF
	40	chmod 0644 "$targetDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup"
	41
	42	# keep us lean by effectively running "apt-get clean" after every install
	43	aptGetClean='"rm -f /var/cache/apt/archives/.deb /var/cache/apt/archives/partial/.deb /var/cache/apt/*.bin \|\| true";'
	44	cat > "$targetDir/etc/apt/apt.conf.d/docker-clean" <<-EOF
	45	# Since for most Docker users, package installs happen in "docker build" steps,
	46	# they essentially become individual layers due to the way Docker handles
	47	# layering, especially using CoW filesystems. What this means for us is that
	48	# the caches that APT keeps end up just wasting space in those layers, making
	49	# our layers unnecessarily large (especially since we'll normally never use
	50	# these caches again and will instead just "docker build" again and make a brand
	51	# new image).
	52
	53	# Ideally, these would just be invoking "apt-get clean", but in our testing,
	54	# that ended up being cyclic and we got stuck on APT's lock, so we get this fun
	55	# creation that's essentially just "apt-get clean".
	56	DPkg::Post-Invoke { $aptGetClean };
	57	APT::Update::Post-Invoke { $aptGetClean };
	58
	59	Dir::Cache::pkgcache "";
	60	Dir::Cache::srcpkgcache "";
	61
	62	# Note that we do realize this isn't the ideal way to do this, and are always
	63	# open to better suggestions (https://github.com/debuerreotype/debuerreotype/issues).
	64	EOF
	65	chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-clean"
	66
	67	# remove apt-cache translations for faster "apt-get update"
	68	cat > "$targetDir/etc/apt/apt.conf.d/docker-no-languages" <<-'EOF'
	69	# In Docker, we don't often need the "Translations" files, so we're just wasting
	70	# time and space by downloading them, and this inhibits that. For users that do
	71	# need them, it's a simple matter to delete this file and "apt-get update". :)
	72
	73	Acquire::Languages "none";
	74	EOF
	75	chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-no-languages"
	76
	77	cat > "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes" <<-'EOF'
	78	# Since Docker users using "RUN apt-get update && apt-get install -y ..." in
	79	# their Dockerfiles don't go delete the lists files afterwards, we want them to
	80	# be as small as possible on-disk, so we explicitly request "gz" versions and
	81	# tell Apt to keep them gzipped on-disk.
	82
	83	# For comparison, an "apt-get update" layer without this on a pristine
	84	# "debian:wheezy" base image was "29.88 MB", where with this it was only
	85	# "8.273 MB".
	86
	87	Acquire::GzipIndexes "true";
	88	Acquire::CompressionTypes::Order:: "gz";
	89	EOF
	90	chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-gzip-indexes"
	91
	92	# update "autoremove" configuration to be aggressive about removing suggests deps that weren't manually installed
	93	cat > "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests" <<-'EOF'
	94	# Since Docker users are looking for the smallest possible final images, the
	95	# following emerges as a very common pattern:
	96
	97	# RUN apt-get update \
	98	# && apt-get install -y <packages> \
	99	# && <do some compilation work> \
	100	# && apt-get purge -y --auto-remove <packages>
	101
	102	# By default, APT will actually _keep_ packages installed via Recommends or
	103	# Depends if another package Suggests them, even and including if the package
	104	# that originally caused them to be installed is removed. Setting this to
	105	# "false" ensures that APT is appropriately aggressive about removing the
	106	# packages it added.
	107
	108	# https://aptitude.alioth.debian.org/doc/en/ch02s05s05.html#configApt-AutoRemove-SuggestsImportant
	109	Apt::AutoRemove::SuggestsImportant "false";
	110	EOF
	111	chmod 0644 "$targetDir/etc/apt/apt.conf.d/docker-autoremove-suggests"

+61

-0

scripts/debuerreotype-slimify less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir>' \
	6	'rootfs'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	[ -n "$targetDir" ]
	10
	11	IFS=$'\n'; set -o noglob
	12	slimExcludes=( $(grep -vE '^#\|^$' "$thisDir/.slimify-excludes" \| sort -u) )
	13	set +o noglob; unset IFS
	14
	15	dpkgCfgFile="$targetDir/etc/dpkg/dpkg.cfg.d/docker"
	16	mkdir -p "$(dirname "$dpkgCfgFile")"
	17	{
	18	echo '# This is the "slim" variant of the Debian base image.'
	19	echo '# Many files which are normally unnecessary in containers are excluded,'
	20	echo '# and this configuration file keeps them that way.'
	21	} > "$dpkgCfgFile"
	22
	23	neverExclude='/usr/share/doc/*/copyright'
	24	for slimExclude in "${slimExcludes[@]}"; do
	25	{
	26	echo
	27	echo "# dpkg -S '$slimExclude'"
	28	if dpkgOutput="$("$thisDir/debuerreotype-chroot" "$targetDir" dpkg -S "$slimExclude" 2>&1)"; then
	29	echo "$dpkgOutput" \| sed 's/: .*//g; s/, /\n/g' \| sort -u \| xargs
	30	else
	31	echo "$dpkgOutput"
	32	fi \| fold -w 76 -s \| sed 's/^/# /'
	33	echo "path-exclude $slimExclude"
	34	} >> "$dpkgCfgFile"
	35
	36	if [[ "$slimExclude" == '/' ]]; then
	37	if [ -d "$targetDir/$(dirname "$slimExclude")" ]; then
	38	# use two passes so that we don't fail trying to remove directories from $neverExclude
	39	"$thisDir/debuerreotype-chroot" "$targetDir" \
	40	find "$(dirname "$slimExclude")" \
	41	-mindepth 1 \
	42	-not -path "$neverExclude" \
	43	-not -type d \
	44	-delete
	45	"$thisDir/debuerreotype-chroot" "$targetDir" \
	46	find "$(dirname "$slimExclude")" \
	47	-mindepth 1 \
	48	-empty \
	49	-delete
	50	fi
	51	else
	52	"$thisDir/debuerreotype-chroot" "$targetDir" rm -f "$slimExclude"
	53	fi
	54	done
	55	{
	56	echo
	57	echo '# always include these files, especially for license compliance'
	58	echo "path-include $neverExclude"
	59	} >> "$dpkgCfgFile"
	60	chmod 0644 "$dpkgCfgFile"

+27

-0

scripts/debuerreotype-tar less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'<target-dir> <target-tar>' \
	6	'rootfs rootfs.tar'
	7
	8	targetDir="${1:-}"; shift \|\| eusage 'missing target-dir'
	9	[ -n "$targetDir" ]
	10	targetTar="${1:-}"; shift \|\| eusage 'missing target-tar'
	11	[ -n "$targetTar" ]
	12
	13	epoch="$(< "$targetDir/debuerreotype-epoch")"
	14	[ -n "$epoch" ]
	15
	16	"$thisDir/debuerreotype-fixup" "$targetDir"
	17	tar --create \
	18	--file "$targetTar" \
	19	--auto-compress \
	20	--directory "$targetDir" \
	21	--exclude-from "$thisDir/.tar-exclude" \
	22	--numeric-owner \
	23	--transform 's,^./,,' \
	24	--sort name \
	25	.
	26	touch --no-dereference --date="@$epoch" "$targetTar"

+9

-0

scripts/debuerreotype-version less more

	0	#!/usr/bin/env bash
	1	set -Eeuo pipefail
	2
	3	thisDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
	4	source "$thisDir/.constants.sh" \
	5	'' \
	6	''
	7
	8	_version