Commit b1697a2171b6474790d2779bb05629aebc4d4948 - golang-github-appc-docker2aci

Merge tag 'upstream/0.11.1+dfsg' Upstream version 0.11.1+dfsg Dmitry Smirnov 7 years ago

12 changed file(s) with 383 addition(s) and 222 deletion(s). Raw diff Collapse all Expand all

+23

-1

CHANGELOG.md less more

	0	## v0.11.1
	1
	2	v0.11.1 is a bugfix release.
	3
	4	- Fix parallel pull synchronisation ([#167](https://github.com/appc/docker2aci/pull/167), [#168](https://github.com/appc/docker2aci/pull/168)).
	5
	6	## v0.11.0
	7
	8	This release splits the `--insecure` flag in two, `--insecure-skip-verify` to skip TLS verification, and `--insecure-allow-http` to allow unencrypted connections when fetching images. It also includes a couple of bugfixes.
	9
	10	- Add missing message to channel on successful layer download ([#161](https://github.com/appc/docker2aci/pull/161)).
	11	- Fix a panic when a layer being fetched encounters an error ([#162](https://github.com/appc/docker2aci/pull/162)).
	12	- Split `--insecure` flag in two ([#163](https://github.com/appc/docker2aci/pull/163)).
	13
	14	## v0.10.0
	15
	16	This release includes two major performance optimizations: parallel layer pull and parallel ACI compression.
	17
	18	- Pull layers in parallel ([#158](https://github.com/appc/docker2aci/pull/158)).
	19	- Use a parallel compression library ([#157](https://github.com/appc/docker2aci/pull/157)).
	20	- Fix auth token parsing to handle services with spaces in their names ([#150](https://github.com/appc/docker2aci/pull/150)).
	21
0	22	## v0.9.3
1	23
2		v0.9.2 is a minor bug fix release.
	24	v0.9.3 is a minor bug fix release.
3	25
4	26	- Use the default transport when doing HTTP requests ([#147](https://github.com/appc/docker2aci/pull/147)). We were using an empty transport which didn't pass on the proxy configuration.
5	27

+65

-23

Godeps/Godeps.json less more

0	0	{
1	1	"ImportPath": "github.com/appc/docker2aci",
2	2	"GoVersion": "go1.6",
	3	"GodepVersion": "v72",
3	4	"Deps": [
4	5	{
5	6	"ImportPath": "github.com/appc/spec/aci",
6		"Comment": "v0.7.4",
7		"Rev": "0875c991a496fe2b65ed296551c667b199a6bfb8"
	7	"Comment": "v0.8.2",
	8	"Rev": "05d50e649f47c4f4bfad9bbf9a95bbde1b26c783"
8	9	},
9	10	{
10	11	"ImportPath": "github.com/appc/spec/pkg/acirenderer",
11		"Comment": "v0.7.4",
12		"Rev": "0875c991a496fe2b65ed296551c667b199a6bfb8"
	12	"Comment": "v0.8.2",
	13	"Rev": "05d50e649f47c4f4bfad9bbf9a95bbde1b26c783"
13	14	},
14	15	{
15	16	"ImportPath": "github.com/appc/spec/pkg/device",
16		"Comment": "v0.7.4",
17		"Rev": "0875c991a496fe2b65ed296551c667b199a6bfb8"
	17	"Comment": "v0.8.2",
	18	"Rev": "05d50e649f47c4f4bfad9bbf9a95bbde1b26c783"
18	19	},
19	20	{
20	21	"ImportPath": "github.com/appc/spec/pkg/tarheader",
21		"Comment": "v0.7.4",
22		"Rev": "0875c991a496fe2b65ed296551c667b199a6bfb8"
	22	"Comment": "v0.8.2",
	23	"Rev": "05d50e649f47c4f4bfad9bbf9a95bbde1b26c783"
23	24	},
24	25	{
25	26	"ImportPath": "github.com/appc/spec/schema",
26		"Comment": "v0.7.4",
27		"Rev": "0875c991a496fe2b65ed296551c667b199a6bfb8"
	27	"Comment": "v0.8.2",
	28	"Rev": "05d50e649f47c4f4bfad9bbf9a95bbde1b26c783"
28	29	},
29	30	{
30	31	"ImportPath": "github.com/appc/spec/schema/common",
31		"Comment": "v0.7.4",
32		"Rev": "0875c991a496fe2b65ed296551c667b199a6bfb8"
	32	"Comment": "v0.8.2",
	33	"Rev": "05d50e649f47c4f4bfad9bbf9a95bbde1b26c783"
33	34	},
34	35	{
35	36	"ImportPath": "github.com/appc/spec/schema/types",
36		"Comment": "v0.7.4",
37		"Rev": "0875c991a496fe2b65ed296551c667b199a6bfb8"
38		},
39		{
40		"ImportPath": "github.com/camlistore/camlistore/pkg/errorutil",
41		"Rev": "9868aa0f8d8a93ff0b30ff0de46cc351b6b88b30"
	37	"Comment": "v0.8.2",
	38	"Rev": "05d50e649f47c4f4bfad9bbf9a95bbde1b26c783"
42	39	},
43	40	{
44	41	"ImportPath": "github.com/coreos/go-semver/semver",

47	44	{
48	45	"ImportPath": "github.com/coreos/ioprogress",
49	46	"Rev": "e7fc03058804de5488baed8df5b89f3924b9ec9a"
	47	},
	48	{
	49	"ImportPath": "github.com/coreos/pkg/progressutil",
	50	"Comment": "v2",
	51	"Rev": "7f080b6c11ac2d2347c3cd7521e810207ea1a041"
50	52	},
51	53	{
52	54	"ImportPath": "github.com/docker/distribution/digest",

59	61	"Rev": "099622876197e3b24d627a72053d1bcc8968076a"
60	62	},
61	63	{
	64	"ImportPath": "github.com/gogo/protobuf/proto",
	65	"Comment": "v0.2",
	66	"Rev": "4168943e65a2802828518e95310aeeed6d84c4e5"
	67	},
	68	{
	69	"ImportPath": "github.com/klauspost/compress/flate",
	70	"Comment": "v1.0",
	71	"Rev": "006acde2c5d283d2f8b8aa03d8f0cd2891c680cf"
	72	},
	73	{
	74	"ImportPath": "github.com/klauspost/cpuid",
	75	"Comment": "v1.0",
	76	"Rev": "09cded8978dc9e80714c4d85b0322337b0a1e5e0"
	77	},
	78	{
	79	"ImportPath": "github.com/klauspost/crc32",
	80	"Comment": "v1.0",
	81	"Rev": "19b0b332c9e4516a6370a0456e6182c3b5036720"
	82	},
	83	{
	84	"ImportPath": "github.com/klauspost/pgzip",
	85	"Comment": "v1.0",
	86	"Rev": "95e8170c5d4da28db9c64dfc9ec3138ea4466fd4"
	87	},
	88	{
62	89	"ImportPath": "github.com/spf13/pflag",
63	90	"Rev": "08b1a584251b5b62f458943640fc8ebd4d50aaa5"
	91	},
	92	{
	93	"ImportPath": "go4.org/errorutil",
	94	"Rev": "03efcb870d84809319ea509714dd6d19a1498483"
64	95	},
65	96	{
66	97	"ImportPath": "golang.org/x/crypto/ssh/terminal",
67	98	"Rev": "a7ead6ddf06233883deca151dffaef2effbf498f"
68	99	},
69	100	{
70		"ImportPath": "k8s.io/kubernetes/pkg/api/resource",
71		"Comment": "v0.12.0-270-g53ec66c",
72		"Rev": "53ec66caf4e952a1384ec93b9f0cde37616e4caf"
	101	"ImportPath": "gopkg.in/inf.v0",
	102	"Comment": "v0.9.0",
	103	"Rev": "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4"
73	104	},
74	105	{
75		"ImportPath": "speter.net/go/exp/math/dec/inf",
76		"Rev": "42ca6cd68aa922bc3f32f1e056e61b65945d9ad7"
	106	"ImportPath": "k8s.io/kubernetes/pkg/api/resource",
	107	"Comment": "v1.3.0-alpha.4-205-g9625926",
	108	"Rev": "9625926852e215caa35e278c2cd7926744532291"
	109	},
	110	{
	111	"ImportPath": "k8s.io/kubernetes/pkg/conversion",
	112	"Comment": "v1.3.0-alpha.4-205-g9625926",
	113	"Rev": "9625926852e215caa35e278c2cd7926744532291"
	114	},
	115	{
	116	"ImportPath": "k8s.io/kubernetes/third_party/forked/reflect",
	117	"Comment": "v1.3.0-alpha.4-205-g9625926",
	118	"Rev": "9625926852e215caa35e278c2cd7926744532291"
77	119	}
78	120	]
79	121	}

-0

lib/common/common.go less more

43	43	Images []string
44	44	}
45	45
	46	// InsecureConfig represents the different insecure options available
	47	type InsecureConfig struct {
	48	SkipVerify bool
	49	AllowHTTP bool
	50	}
	51
46	52	func (e *ErrSeveralImages) Error() string {
47	53	return e.Msg
48	54	}

+18

-24

lib/docker2aci.go less more

17	17
18	18	import (
19	19	"archive/tar"
20		"compress/gzip"
21	20	"fmt"
22	21	"io"
23	22	"io/ioutil"

37	36	"github.com/appc/spec/pkg/acirenderer"
38	37	"github.com/appc/spec/schema"
39	38	appctypes "github.com/appc/spec/schema/types"
	39	gzip "github.com/klauspost/pgzip"
40	40	)
41	41
42	42	// CommonConfig represents the shared configuration options for converting

52	52	// converting Docker images.
53	53	type RemoteConfig struct {
54	54	CommonConfig
55		Username string // username to use if the image to convert needs authentication
56		Password string // password to use if the image to convert needs authentication
57		Insecure bool // allow converting from insecure repos
	55	Username string // username to use if the image to convert needs authentication
	56	Password string // password to use if the image to convert needs authentication
	57	Insecure common.InsecureConfig // Insecure options
58	58	}
59	59
60	60	// FileConfig represents the saved file specific configuration for converting

123	123
124	124	conversionStore := newConversionStore()
125	125
	126	// only compress individual layers if we're not squashing
	127	layerCompression := compression
	128	if squash {
	129	layerCompression = common.NoCompression
	130	}
	131
	132	aciLayerPaths, aciManifests, err := backend.BuildACI(ancestry, parsedDockerURL, layersOutputDir, tmpDir, layerCompression)
	133	if err != nil {
	134	return nil, err
	135	}
	136
126	137	var images acirenderer.Images
127		var aciLayerPaths []string
128		var curPwl []string
129		for i := len(ancestry) - 1; i >= 0; i-- {
130		layerID := ancestry[i]
131
132		// only compress individual layers if we're not squashing
133		layerCompression := compression
134		if squash {
135		layerCompression = common.NoCompression
136		}
137
138		aciPath, manifest, err := backend.BuildACI(i, layerID, parsedDockerURL, layersOutputDir, tmpDir, curPwl, layerCompression)
139		if err != nil {
140		return nil, fmt.Errorf("error building layer: %v", err)
141		}
142
143		key, err := conversionStore.WriteACI(aciPath)
	138	for i, aciLayerPath := range aciLayerPaths {
	139	key, err := conversionStore.WriteACI(aciLayerPath)
144	140	if err != nil {
145	141	return nil, fmt.Errorf("error inserting in the conversion store: %v", err)
146	142	}
147	143
148		images = append(images, acirenderer.Image{Im: manifest, Key: key, Level: uint16(i)})
149		aciLayerPaths = append(aciLayerPaths, aciPath)
150		curPwl = manifest.PathWhitelist
	144	images = append(images, acirenderer.Image{Im: aciManifests[i], Key: key, Level: uint16(len(aciLayerPaths) - 1 - i)})
151	145	}
152	146
153	147	// acirenderer expects images in order from upper to base layer

+42

-33

lib/internal/backend/file/file.go less more

67	67	return ancestry, parsedDockerURL, nil
68	68	}
69	69
70		func (lb FileBackend) BuildACI(layerNumber int, layerID string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) {
71		tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
72		if err != nil {
73		return "", nil, fmt.Errorf("error creating dir: %v", err)
74		}
75		defer os.RemoveAll(tmpDir)
76
77		j, err := getJson(lb.file, layerID)
78		if err != nil {
79		return "", nil, fmt.Errorf("error getting image json: %v", err)
80		}
81
82		layerData := types.DockerImageData{}
83		if err := json.Unmarshal(j, &layerData); err != nil {
84		return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err)
85		}
86
87		tmpLayerPath := path.Join(tmpDir, layerID)
88		tmpLayerPath += ".tar"
89
90		layerFile, err := extractEmbeddedLayer(lb.file, layerID, tmpLayerPath)
91		if err != nil {
92		return "", nil, fmt.Errorf("error getting layer from file: %v", err)
93		}
94		defer layerFile.Close()
95
96		log.Debug("Generating layer ACI...")
97		aciPath, manifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression)
98		if err != nil {
99		return "", nil, fmt.Errorf("error generating ACI: %v", err)
100		}
101
102		return aciPath, manifest, nil
	70	func (lb FileBackend) BuildACI(layerIDs []string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
	71	var aciLayerPaths []string
	72	var aciManifests []*schema.ImageManifest
	73	var curPwl []string
	74	for i := len(layerIDs) - 1; i >= 0; i-- {
	75	tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
	76	if err != nil {
	77	return nil, nil, fmt.Errorf("error creating dir: %v", err)
	78	}
	79	defer os.RemoveAll(tmpDir)
	80
	81	j, err := getJson(lb.file, layerIDs[i])
	82	if err != nil {
	83	return nil, nil, fmt.Errorf("error getting image json: %v", err)
	84	}
	85
	86	layerData := types.DockerImageData{}
	87	if err := json.Unmarshal(j, &layerData); err != nil {
	88	return nil, nil, fmt.Errorf("error unmarshaling layer data: %v", err)
	89	}
	90
	91	tmpLayerPath := path.Join(tmpDir, layerIDs[i])
	92	tmpLayerPath += ".tar"
	93
	94	layerFile, err := extractEmbeddedLayer(lb.file, layerIDs[i], tmpLayerPath)
	95	if err != nil {
	96	return nil, nil, fmt.Errorf("error getting layer from file: %v", err)
	97	}
	98	defer layerFile.Close()
	99
	100	log.Debug("Generating layer ACI...")
	101	aciPath, manifest, err := internal.GenerateACI(i, layerData, dockerURL, outputDir, layerFile, curPwl, compression)
	102	if err != nil {
	103	return nil, nil, fmt.Errorf("error generating ACI: %v", err)
	104	}
	105
	106	aciLayerPaths = append(aciLayerPaths, aciPath)
	107	aciManifests = append(aciManifests, manifest)
	108	curPwl = manifest.PathWhitelist
	109	}
	110
	111	return aciLayerPaths, aciManifests, nil
103	112	}
104	113
105	114	func getImageID(file os.File, dockerURL types.ParsedDockerURL) (string, *types.ParsedDockerURL, error) {

-7

lib/internal/backend/repository/repository.go less more

41	41	repoData *RepoData
42	42	username string
43	43	password string
44		insecure bool
	44	insecure common.InsecureConfig
45	45	hostsV2Support map[string]bool
46	46	hostsV2AuthTokens map[string]map[string]string
47	47	schema string
48	48	imageManifests map[types.ParsedDockerURL]v2Manifest
49	49	}
50	50
51		func NewRepositoryBackend(username string, password string, insecure bool) *RepositoryBackend {
	51	func NewRepositoryBackend(username string, password string, insecure common.InsecureConfig) *RepositoryBackend {
52	52	return &RepositoryBackend{
53	53	username: username,
54	54	password: password,

92	92	}
93	93	}
94	94
95		func (rb RepositoryBackend) BuildACI(layerNumber int, layerID string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) {
	95	func (rb RepositoryBackend) BuildACI(layerIDs []string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
96	96	if rb.hostsV2Support[dockerURL.IndexURL] {
97		return rb.buildACIV2(layerNumber, layerID, dockerURL, outputDir, tmpBaseDir, curPwl, compression)
	97	return rb.buildACIV2(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
98	98	} else {
99		return rb.buildACIV1(layerNumber, layerID, dockerURL, outputDir, tmpBaseDir, curPwl, compression)
	99	return rb.buildACIV1(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
100	100	}
101	101	}
102	102

136	136
137	137	rb.setBasicAuth(req)
138	138
139		client := util.GetTLSClient(rb.insecure)
	139	client := util.GetTLSClient(rb.insecure.SkipVerify)
140	140	res, err = client.Do(req)
141	141	return
142	142	}

148	148	defer res.Body.Close()
149	149	}
150	150	if err != nil \|\| !ok {
151		if rb.insecure {
	151	if rb.insecure.AllowHTTP {
152	152	schema = "http"
153	153	res, err = fetch(schema)
154	154	if err == nil {

+74

-35

lib/internal/backend/repository/repository1.go less more

28	28	"github.com/appc/docker2aci/lib/common"
29	29	"github.com/appc/docker2aci/lib/internal"
30	30	"github.com/appc/docker2aci/lib/internal/types"
	31	"github.com/appc/docker2aci/lib/internal/util"
31	32	"github.com/appc/docker2aci/pkg/log"
32	33	"github.com/appc/spec/schema"
33	34	"github.com/coreos/ioprogress"

61	62	return ancestry, dockerURL, nil
62	63	}
63	64
64		func (rb RepositoryBackend) buildACIV1(layerNumber int, layerID string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) {
65		tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
66		if err != nil {
67		return "", nil, fmt.Errorf("error creating dir: %v", err)
68		}
69		defer os.RemoveAll(tmpDir)
70
71		j, size, err := rb.getJsonV1(layerID, rb.repoData.Endpoints[0], rb.repoData)
72		if err != nil {
73		return "", nil, fmt.Errorf("error getting image json: %v", err)
74		}
75
76		layerData := types.DockerImageData{}
77		if err := json.Unmarshal(j, &layerData); err != nil {
78		return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err)
79		}
80
81		layerFile, err := rb.getLayerV1(layerID, rb.repoData.Endpoints[0], rb.repoData, size, tmpDir)
82		if err != nil {
83		return "", nil, fmt.Errorf("error getting the remote layer: %v", err)
84		}
85		defer layerFile.Close()
86
87		log.Debug("Generating layer ACI...")
88		aciPath, manifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression)
89		if err != nil {
90		return "", nil, fmt.Errorf("error generating ACI: %v", err)
91		}
92
93		return aciPath, manifest, nil
	65	func (rb RepositoryBackend) buildACIV1(layerIDs []string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
	66	layerFiles := make([]*os.File, len(layerIDs))
	67	layerDatas := make([]types.DockerImageData, len(layerIDs))
	68
	69	tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
	70	if err != nil {
	71	return nil, nil, err
	72	}
	73	defer os.RemoveAll(tmpParentDir)
	74
	75	var doneChannels []chan error
	76	for i, layerID := range layerIDs {
	77	doneChan := make(chan error)
	78	doneChannels = append(doneChannels, doneChan)
	79	// https://github.com/golang/go/wiki/CommonMistakes
	80	i := i // golang--
	81	layerID := layerID
	82	go func() {
	83	tmpDir, err := ioutil.TempDir(tmpParentDir, "")
	84	if err != nil {
	85	doneChan <- fmt.Errorf("error creating dir: %v", err)
	86	return
	87	}
	88
	89	j, size, err := rb.getJsonV1(layerID, rb.repoData.Endpoints[0], rb.repoData)
	90	if err != nil {
	91	doneChan <- fmt.Errorf("error getting image json: %v", err)
	92	return
	93	}
	94
	95	layerDatas[i] = types.DockerImageData{}
	96	if err := json.Unmarshal(j, &layerDatas[i]); err != nil {
	97	doneChan <- fmt.Errorf("error unmarshaling layer data: %v", err)
	98	return
	99	}
	100
	101	layerFiles[i], err = rb.getLayerV1(layerID, rb.repoData.Endpoints[0], rb.repoData, size, tmpDir)
	102	if err != nil {
	103	doneChan <- fmt.Errorf("error getting the remote layer: %v", err)
	104	return
	105	}
	106	doneChan <- nil
	107	}()
	108	}
	109	for _, doneChan := range doneChannels {
	110	err := <-doneChan
	111	if err != nil {
	112	return nil, nil, err
	113	}
	114	}
	115	var aciLayerPaths []string
	116	var aciManifests []*schema.ImageManifest
	117	var curPwl []string
	118
	119	for i := len(layerIDs) - 1; i >= 0; i-- {
	120	log.Debug("Generating layer ACI...")
	121	aciPath, manifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression)
	122	if err != nil {
	123	return nil, nil, fmt.Errorf("error generating ACI: %v", err)
	124	}
	125	aciLayerPaths = append(aciLayerPaths, aciPath)
	126	aciManifests = append(aciManifests, manifest)
	127	curPwl = manifest.PathWhitelist
	128
	129	layerFiles[i].Close()
	130	}
	131
	132	return aciLayerPaths, aciManifests, nil
94	133	}
95	134
96	135	func (rb RepositoryBackend) getRepoDataV1(indexURL string, remote string) (RepoData, error) {
97		client := &http.Client{}
	136	client := util.GetTLSClient(rb.insecure.SkipVerify)
98	137	repositoryURL := rb.schema + path.Join(indexURL, "v1", "repositories", remote, "images")
99	138
100	139	req, err := http.NewRequest("GET", repositoryURL, nil)

144	183	}
145	184
146	185	func (rb RepositoryBackend) getImageIDFromTagV1(registry string, appName string, tag string, repoData RepoData) (string, error) {
147		client := &http.Client{}
	186	client := util.GetTLSClient(rb.insecure.SkipVerify)
148	187	// we get all the tags instead of directly getting the imageID of the
149	188	// requested one (.../tags/TAG) because even though it's specified in the
150	189	// Docker API, some registries (e.g. Google Container Registry) don't

187	226	}
188	227
189	228	func (rb RepositoryBackend) getAncestryV1(imgID, registry string, repoData RepoData) ([]string, error) {
190		client := &http.Client{}
	229	client := util.GetTLSClient(rb.insecure.SkipVerify)
191	230	req, err := http.NewRequest("GET", rb.schema+path.Join(registry, "images", imgID, "ancestry"), nil)
192	231	if err != nil {
193	232	return nil, err

220	259	}
221	260
222	261	func (rb RepositoryBackend) getJsonV1(imgID, registry string, repoData RepoData) ([]byte, int64, error) {
223		client := &http.Client{}
	262	client := util.GetTLSClient(rb.insecure.SkipVerify)
224	263	req, err := http.NewRequest("GET", rb.schema+path.Join(registry, "images", imgID, "json"), nil)
225	264	if err != nil {
226	265	return nil, -1, err

255	294	}
256	295
257	296	func (rb RepositoryBackend) getLayerV1(imgID, registry string, repoData RepoData, imgSize int64, tmpDir string) (*os.File, error) {
258		client := &http.Client{}
	297	client := util.GetTLSClient(rb.insecure.SkipVerify)
259	298	req, err := http.NewRequest("GET", rb.schema+path.Join(registry, "images", imgID, "layer"), nil)
260	299	if err != nil {
261	300	return nil, err

+130

-88

lib/internal/backend/repository/repository2.go less more

25	25	"regexp"
26	26	"strconv"
27	27	"strings"
	28	"sync"
28	29	"time"
29	30
30	31	"github.com/appc/docker2aci/lib/common"

33	34	"github.com/appc/docker2aci/lib/internal/util"
34	35	"github.com/appc/docker2aci/pkg/log"
35	36	"github.com/appc/spec/schema"
36		"github.com/coreos/ioprogress"
	37	"github.com/coreos/pkg/progressutil"
37	38	)
38	39
39	40	const (

65	66	return layers, dockerURL, nil
66	67	}
67	68
68		func (rb RepositoryBackend) buildACIV2(layerNumber int, layerID string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) {
69		manifest := rb.imageManifests[*dockerURL]
70
71		layerIndex, err := getLayerIndex(layerID, manifest)
72		if err != nil {
73		return "", nil, err
74		}
75
76		if len(manifest.History) <= layerIndex {
77		return "", nil, fmt.Errorf("history not found for layer %s", layerID)
78		}
79
80		layerData := types.DockerImageData{}
81		if err := json.Unmarshal([]byte(manifest.History[layerIndex].V1Compatibility), &layerData); err != nil {
82		return "", nil, fmt.Errorf("error unmarshaling layer data: %v", err)
83		}
84
85		tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
86		if err != nil {
87		return "", nil, fmt.Errorf("error creating dir: %v", err)
88		}
89		defer os.RemoveAll(tmpDir)
90
91		layerFile, err := rb.getLayerV2(layerID, dockerURL, tmpDir)
92		if err != nil {
93		return "", nil, fmt.Errorf("error getting the remote layer: %v", err)
94		}
95		defer layerFile.Close()
96
97		log.Debug("Generating layer ACI...")
98		aciPath, aciManifest, err := internal.GenerateACI(layerNumber, layerData, dockerURL, outputDir, layerFile, curPwl, compression)
99		if err != nil {
100		return "", nil, fmt.Errorf("error generating ACI: %v", err)
101		}
102
103		return aciPath, aciManifest, nil
	69	func (rb RepositoryBackend) buildACIV2(layerIDs []string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
	70	layerFiles := make([]*os.File, len(layerIDs))
	71	layerDatas := make([]types.DockerImageData, len(layerIDs))
	72
	73	tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-")
	74	if err != nil {
	75	return nil, nil, err
	76	}
	77	defer os.RemoveAll(tmpParentDir)
	78
	79	copier := progressutil.NewCopyProgressPrinter()
	80
	81	var errChannels []chan error
	82	closers := make([]io.ReadCloser, len(layerIDs))
	83	var wg sync.WaitGroup
	84	for i, layerID := range layerIDs {
	85	wg.Add(1)
	86	errChan := make(chan error, 1)
	87	errChannels = append(errChannels, errChan)
	88	// https://github.com/golang/go/wiki/CommonMistakes
	89	i := i // golang--
	90	layerID := layerID
	91	go func() {
	92	defer wg.Done()
	93
	94	manifest := rb.imageManifests[*dockerURL]
	95
	96	layerIndex, err := getLayerIndex(layerID, manifest)
	97	if err != nil {
	98	errChan <- err
	99	return
	100	}
	101
	102	if len(manifest.History) <= layerIndex {
	103	errChan <- fmt.Errorf("history not found for layer %s", layerID)
	104	return
	105	}
	106
	107	layerDatas[i] = types.DockerImageData{}
	108	if err := json.Unmarshal([]byte(manifest.History[layerIndex].V1Compatibility), &layerDatas[i]); err != nil {
	109	errChan <- fmt.Errorf("error unmarshaling layer data: %v", err)
	110	return
	111	}
	112
	113	tmpDir, err := ioutil.TempDir(tmpParentDir, "")
	114	if err != nil {
	115	errChan <- fmt.Errorf("error creating dir: %v", err)
	116	return
	117	}
	118
	119	layerFiles[i], closers[i], err = rb.getLayerV2(layerID, dockerURL, tmpDir, copier)
	120	if err != nil {
	121	errChan <- fmt.Errorf("error getting the remote layer: %v", err)
	122	return
	123	}
	124	errChan <- nil
	125	}()
	126	}
	127	// Need to wait for all of the readers to be added to the copier (which happens during rb.getLayerV2)
	128	wg.Wait()
	129	err = copier.PrintAndWait(os.Stderr, 500*time.Millisecond, nil)
	130	if err != nil {
	131	return nil, nil, err
	132	}
	133	for _, closer := range closers {
	134	if closer != nil {
	135	closer.Close()
	136	}
	137	}
	138	for _, errChan := range errChannels {
	139	err := <-errChan
	140	if err != nil {
	141	return nil, nil, err
	142	}
	143	}
	144	for _, layerFile := range layerFiles {
	145	err := layerFile.Sync()
	146	if err != nil {
	147	return nil, nil, err
	148	}
	149	}
	150	var aciLayerPaths []string
	151	var aciManifests []*schema.ImageManifest
	152	var curPwl []string
	153	for i := len(layerIDs) - 1; i >= 0; i-- {
	154	log.Debug("Generating layer ACI...")
	155	aciPath, aciManifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression)
	156	if err != nil {
	157	return nil, nil, fmt.Errorf("error generating ACI: %v", err)
	158	}
	159	aciLayerPaths = append(aciLayerPaths, aciPath)
	160	aciManifests = append(aciManifests, aciManifest)
	161	curPwl = aciManifest.PathWhitelist
	162
	163	layerFiles[i].Close()
	164	}
	165
	166	return aciLayerPaths, aciManifests, nil
104	167	}
105	168
106	169	func (rb RepositoryBackend) getManifestV2(dockerURL types.ParsedDockerURL) (*v2Manifest, []string, error) {

223	286	return -1, fmt.Errorf("layer not found in manifest: %s", layerID)
224	287	}
225	288
226		func (rb RepositoryBackend) getLayerV2(layerID string, dockerURL types.ParsedDockerURL, tmpDir string) (*os.File, error) {
	289	func (rb RepositoryBackend) getLayerV2(layerID string, dockerURL types.ParsedDockerURL, tmpDir string, copier progressutil.CopyProgressPrinter) (os.File, io.ReadCloser, error) {
227	290	url := rb.schema + path.Join(dockerURL.IndexURL, "v2", dockerURL.ImageName, "blobs", layerID)
228	291	req, err := http.NewRequest("GET", url, nil)
229	292	if err != nil {
230		return nil, err
	293	return nil, nil, err
231	294	}
232	295
233	296	rb.setBasicAuth(req)
234	297
235	298	res, err := rb.makeRequest(req, dockerURL.ImageName)
236	299	if err != nil {
237		return nil, err
238		}
239		defer res.Body.Close()
	300	return nil, nil, err
	301	}
240	302
241	303	if res.StatusCode == http.StatusTemporaryRedirect \|\| res.StatusCode == http.StatusFound {
242	304	location := res.Header.Get("Location")
243	305	if location != "" {
244	306	req, err = http.NewRequest("GET", location, nil)
245	307	if err != nil {
246		return nil, err
	308	return nil, nil, err
247	309	}
248	310	res, err = rb.makeRequest(req, dockerURL.ImageName)
249	311	if err != nil {
250		return nil, err
	312	return nil, nil, err
251	313	}
252	314	defer res.Body.Close()
253	315	}
254	316	}
255	317
256	318	if res.StatusCode != http.StatusOK {
257		return nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL)
	319	return nil, nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL)
258	320	}
259	321
260	322	var in io.Reader
261	323	in = res.Body
262	324
	325	var size int64
	326
263	327	if hdr := res.Header.Get("Content-Length"); hdr != "" {
264		imgSize, err := strconv.ParseInt(hdr, 10, 64)
	328	size, err = strconv.ParseInt(hdr, 10, 64)
265	329	if err != nil {
266		return nil, err
267		}
268
269		prefix := "Downloading " + layerID[:18]
270		fmtBytesSize := 18
271		barSize := int64(80 - len(prefix) - fmtBytesSize)
272		bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr)
273		fmtfunc := func(progress, total int64) string {
274		return fmt.Sprintf(
275		"%s: %s %s",
276		prefix,
277		bar(progress, total),
278		ioprogress.DrawTextFormatBytes(progress, total),
279		)
280		}
281		in = &ioprogress.Reader{
282		Reader: res.Body,
283		Size: imgSize,
284		DrawFunc: ioprogress.DrawTerminalf(os.Stderr, fmtfunc),
285		DrawInterval: 500 * time.Millisecond,
286		}
287		}
	330	return nil, nil, err
	331	}
	332	}
	333
	334	name := "Downloading " + layerID[:18]
288	335
289	336	layerFile, err := ioutil.TempFile(tmpDir, "dockerlayer-")
290	337	if err != nil {
291		return nil, err
292		}
293
294		_, err = io.Copy(layerFile, in)
295		if err != nil {
296		return nil, err
297		}
298
299		if err := layerFile.Sync(); err != nil {
300		return nil, err
301		}
302
303		return layerFile, nil
	338	return nil, nil, err
	339	}
	340
	341	err = copier.AddCopy(in, name, size, layerFile)
	342	if err != nil {
	343	return nil, nil, err
	344	}
	345
	346	return layerFile, res.Body, nil
304	347	}
305	348
306	349	func (rb RepositoryBackend) makeRequest(req http.Request, repo string) (*http.Response, error) {

314	357	}
315	358	}
316	359
317		client := util.GetTLSClient(rb.insecure)
	360	client := util.GetTLSClient(rb.insecure.SkipVerify)
318	361	res, err := client.Do(req)
319	362	if err != nil {
320	363	return nil, err

329	372	return res, err
330	373	}
331	374
332		tokens := strings.Split(hdr, " ")
333		if len(tokens) != 2 \|\| strings.ToLower(tokens[0]) != "bearer" {
	375	tokens := strings.Split(hdr, ",")
	376	if len(tokens) != 3 \|\|
	377	!strings.HasPrefix(strings.ToLower(tokens[0]), "bearer realm") {
334	378	return res, err
335	379	}
336	380	res.Body.Close()
337
338		tokens = strings.Split(tokens[1], ",")
339	381
340	382	var realm, service, scope string
341	383	for _, token := range tokens {
342		if strings.HasPrefix(token, "realm") {
343		realm = strings.Trim(token[len("realm="):], "\"")
	384	if strings.HasPrefix(strings.ToLower(token), "bearer realm") {
	385	realm = strings.Trim(token[len("bearer realm="):], "\"")
344	386	}
345	387	if strings.HasPrefix(token, "service") {
346	388	service = strings.Trim(token[len("service="):], "\"")

-2

lib/internal/internal.go less more

19	19
20	20	import (
21	21	"archive/tar"
22		"compress/gzip"
23	22	"encoding/json"
24	23	"fmt"
25	24	"io"

39	38	"github.com/appc/spec/aci"
40	39	"github.com/appc/spec/schema"
41	40	appctypes "github.com/appc/spec/schema/types"
	41	gzip "github.com/klauspost/pgzip"
42	42	)
43	43
44	44	// Docker2ACIBackend is the interface that abstracts converting Docker layers

51	51	// path and its converted ImageManifest.
52	52	type Docker2ACIBackend interface {
53	53	GetImageInfo(dockerUrl string) ([]string, *types.ParsedDockerURL, error)
54		BuildACI(layerNumber int, layerID string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, curPWl []string, compression common.Compression) (string, schema.ImageManifest, error)
	54	BuildACI(layerIDs []string, dockerURL types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []schema.ImageManifest, error)
55	55	}
56	56
57	57	// GenerateACI takes a Docker layer and generates an ACI from it.

-1

lib/version.go less more

15	15
16	16	import "github.com/appc/spec/schema"
17	17
18		var Version = "0.9.3"
	18	var Version = "0.11.1"
19	19	var AppcVersion = schema.AppContainerVersion

+13

-8

main.go less more

29	29	)
30	30
31	31	var (
32		flagNoSquash bool
33		flagImage string
34		flagDebug bool
35		flagInsecure bool
36		flagCompression string
37		flagVersion bool
	32	flagNoSquash bool
	33	flagImage string
	34	flagDebug bool
	35	flagInsecureSkipVerify bool
	36	flagInsecureAllowHTTP bool
	37	flagCompression string
	38	flagVersion bool
38	39	)
39	40
40	41	func init() {
41	42	flag.BoolVar(&flagNoSquash, "nosquash", false, "Don't squash layers and output every layer as ACI")
42	43	flag.StringVar(&flagImage, "image", "", "When converting a local file, it selects a particular image to convert. Format: IMAGE_NAME[:TAG]")
43	44	flag.BoolVar(&flagDebug, "debug", false, "Enables debug messages")
44		flag.BoolVar(&flagInsecure, "insecure", false, "Uses unencrypted connections when fetching images")
	45	flag.BoolVar(&flagInsecureSkipVerify, "insecure-skip-verify", false, "Don't verify certificates when fetching images")
	46	flag.BoolVar(&flagInsecureAllowHTTP, "insecure-allow-http", false, "Uses unencrypted connections when fetching images")
45	47	flag.StringVar(&flagCompression, "compression", "gzip", "Type of compression to use; allowed values: gzip, none")
46	48	flag.BoolVar(&flagVersion, "version", false, "Print version")
47	49	}

98	100	CommonConfig: cfg,
99	101	Username: username,
100	102	Password: password,
101		Insecure: flagInsecure,
	103	Insecure: common.InsecureConfig{
	104	SkipVerify: flagInsecureSkipVerify,
	105	AllowHTTP: flagInsecureAllowHTTP,
	106	},
102	107	}
103	108
104	109	aciLayerPaths, err = docker2aci.ConvertRemoteRepo(dockerURL, remoteConfig)

-0

tests/test.sh less more

40	40
41	41	echo "### Test case ${TESTNAME}: converting to ACI..."
42	42	sudo docker save -o ${TESTNAME}.docker $PREFIX/${TESTNAME}
	43	# Docker now writes files as root, so make them readable
	44	sudo chmod o+rx ${TESTNAME}.docker
43	45	$DOCKER2ACI ${TESTNAME}.docker
44	46
45	47	echo "### Test case ${TESTNAME}: test in rkt..."