Codebase list golang-golang-x-mod / 98d530a
zip: add CreateFromVCS, which creates a module zip from vcs Updates golang/go#37413 Change-Id: I5ea07a6e4eedc6cb215e4893944f1ab215ea8f2b Reviewed-on: https://go-review.googlesource.com/c/mod/+/330769 Trust: Jean de Klerk <deklerk@google.com> Trust: Jay Conrod <jayconrod@google.com> Run-TryBot: Jean de Klerk <deklerk@google.com> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Jay Conrod <jayconrod@google.com> Jean de Klerk 2 years ago
2 changed file(s) with 443 addition(s) and 27 deletion(s). Raw diff Collapse all Expand all
+121
-1
zip/zip.go less more
5252 "io"
5353 "io/ioutil"
5454 "os"
55 "os/exec"
5556 "path"
5657 "path/filepath"
5758 "strings"
554555 if zerr, ok := err.(*zipError); ok {
555556 zerr.path = dir
556557 } else if err != nil {
557 err = &zipError{verb: "create zip", path: dir, err: err}
558 err = &zipError{verb: "create zip from directory", path: dir, err: err}
558559 }
559560 }()
560561
566567 return Create(w, m, files)
567568 }
568569
570 // CreateFromVCS creates a module zip file for module m from the contents of a
571 // VCS repository stored locally. The zip content is written to w.
572 //
573 // repo must be an absolute path to the base of the repository, such as
574 // "/Users/some-user/my-repo".
575 //
576 // revision is the revision of the repository to create the zip from. Examples
577 // include HEAD or SHA sums for git repositories.
578 //
579 // subdir must be the relative path from the base of the repository, such as
580 // "sub/dir". To create a zip from the base of the repository, pass an empty
581 // string.
582 func CreateFromVCS(w io.Writer, m module.Version, repo, revision, subdir string) (err error) {
583 defer func() {
584 if zerr, ok := err.(*zipError); ok {
585 zerr.path = repo
586 } else if err != nil {
587 err = &zipError{verb: "create zip from version control system", path: repo, err: err}
588 }
589 }()
590
591 var filesToCreate []File
592
593 switch {
594 case isGitRepo(repo):
595 files, err := filesInGitRepo(repo, revision, subdir)
596 if err != nil {
597 return err
598 }
599
600 filesToCreate = files
601 default:
602 return fmt.Errorf("%q does not use a recognised version control system", repo)
603 }
604
605 return Create(w, m, filesToCreate)
606 }
607
608 // filterGitIgnored filters out any files that are git ignored in the directory.
609 func filesInGitRepo(dir, rev, subdir string) ([]File, error) {
610 stderr := bytes.Buffer{}
611 stdout := bytes.Buffer{}
612
613 // Incredibly, git produces different archives depending on whether
614 // it is running on a Windows system or not, in an attempt to normalize
615 // text file line endings. Setting -c core.autocrlf=input means only
616 // translate files on the way into the repo, not on the way out (archive).
617 // The -c core.eol=lf should be unnecessary but set it anyway.
618 //
619 // Note: We use git archive to understand which files are actually included,
620 // ignoring things like .gitignore'd files. We could also use other
621 // techniques like git ls-files, but this approach most closely matches what
622 // the Go command does, which is beneficial.
623 //
624 // Note: some of this code copied from https://go.googlesource.com/go/+/refs/tags/go1.16.5/src/cmd/go/internal/modfetch/codehost/git.go#826.
625 cmd := exec.Command("git", "-c", "core.autocrlf=input", "-c", "core.eol=lf", "archive", "--format=zip", rev)
626 if subdir != "" {
627 cmd.Args = append(cmd.Args, subdir)
628 }
629 cmd.Dir = dir
630 cmd.Stdout = &stdout
631 cmd.Stderr = &stderr
632 if err := cmd.Run(); err != nil {
633 return nil, fmt.Errorf("error running `git archive`: %w, %s", err, stderr.String())
634 }
635
636 rawReader := bytes.NewReader(stdout.Bytes())
637 zipReader, err := zip.NewReader(rawReader, int64(stdout.Len()))
638 if err != nil {
639 return nil, err
640 }
641
642 var fs []File
643 for _, zf := range zipReader.File {
644 if !strings.HasPrefix(zf.Name, subdir) || strings.HasSuffix(zf.Name, "/") {
645 continue
646 }
647
648 n := strings.TrimPrefix(zf.Name, subdir)
649 if n == "" {
650 continue
651 }
652 n = strings.TrimPrefix(n, string(filepath.Separator))
653
654 fs = append(fs, zipFile{
655 name: n,
656 f: zf,
657 })
658 }
659
660 return fs, nil
661 }
662
663 // isGitRepo reports whether the given directory is a git repo.
664 func isGitRepo(dir string) bool {
665 stdout := &bytes.Buffer{}
666 cmd := exec.Command("git", "rev-parse", "--git-dir")
667 cmd.Dir = dir
668 cmd.Stdout = stdout
669 if err := cmd.Run(); err != nil {
670 return false
671 }
672 gitDir := strings.TrimSpace(string(stdout.Bytes()))
673 if !filepath.IsAbs(gitDir) {
674 gitDir = filepath.Join(dir, gitDir)
675 }
676 wantDir := filepath.Join(dir, ".git")
677 return wantDir == gitDir
678 }
679
569680 type dirFile struct {
570681 filePath, slashPath string
571682 info os.FileInfo
574685 func (f dirFile) Path() string { return f.slashPath }
575686 func (f dirFile) Lstat() (os.FileInfo, error) { return f.info, nil }
576687 func (f dirFile) Open() (io.ReadCloser, error) { return os.Open(f.filePath) }
688
689 type zipFile struct {
690 name string
691 f *zip.File
692 }
693
694 func (f zipFile) Path() string { return f.name }
695 func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil }
696 func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
577697
578698 // isVendoredPackage attempts to report whether the given filename is contained
579699 // in a package whose import path contains (but does not end with) the component
+322
-26
zip/zip_test.go less more
7777 return test, nil
7878 }
7979
80 func extractTxtarToTempDir(arc *txtar.Archive) (dir string, err error) {
80 func extractTxtarToTempDir(arc *txtar.Archive) (dir string, cleanup func(), err error) {
8181 dir, err = ioutil.TempDir("", "zip_test-*")
8282 if err != nil {
83 return "", err
83 return "", func() {}, err
84 }
85 cleanup = func() {
86 os.RemoveAll(dir)
8487 }
8588 defer func() {
8689 if err != nil {
87 os.RemoveAll(dir)
90 cleanup()
8891 }
8992 }()
9093 for _, f := range arc.Files {
9194 filePath := filepath.Join(dir, f.Name)
9295 if err := os.MkdirAll(filepath.Dir(filePath), 0777); err != nil {
93 return "", err
96 return "", func() {}, err
9497 }
9598 if err := ioutil.WriteFile(filePath, f.Data, 0666); err != nil {
96 return "", err
97 }
98 }
99 return dir, nil
99 return "", func() {}, err
100 }
101 }
102 return dir, cleanup, nil
100103 }
101104
102105 func extractTxtarToTempZip(arc *txtar.Archive) (zipPath string, err error) {
268271 break
269272 }
270273 }
271 tmpDir, err := extractTxtarToTempDir(test.archive)
272 if err != nil {
273 t.Fatal(err)
274 }
275 defer func() {
276 if err := os.RemoveAll(tmpDir); err != nil {
277 t.Errorf("removing temp directory: %v", err)
278 }
279 }()
274 tmpDir, cleanup, err := extractTxtarToTempDir(test.archive)
275 if err != nil {
276 t.Fatal(err)
277 }
278 defer cleanup()
280279
281280 // Check the directory.
282281 cf, err := modzip.CheckDir(tmpDir)
460459 }
461460
462461 // Write files to a temporary directory.
463 tmpDir, err := extractTxtarToTempDir(test.archive)
464 if err != nil {
465 t.Fatal(err)
466 }
467 defer func() {
468 if err := os.RemoveAll(tmpDir); err != nil {
469 t.Errorf("removing temp directory: %v", err)
470 }
471 }()
462 tmpDir, cleanup, err := extractTxtarToTempDir(test.archive)
463 if err != nil {
464 t.Fatal(err)
465 }
466 defer cleanup()
472467
473468 // Create zip from the directory.
474469 tmpZip, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
14671462 func (f zipFile) Path() string { return f.name }
14681463 func (f zipFile) Lstat() (os.FileInfo, error) { return f.f.FileInfo(), nil }
14691464 func (f zipFile) Open() (io.ReadCloser, error) { return f.f.Open() }
1465
1466 func TestCreateFromVCS_basic(t *testing.T) {
1467 // Write files to a temporary directory.
1468 tmpDir, cleanup, err := extractTxtarToTempDir(txtar.Parse([]byte(`-- go.mod --
1469 module example.com/foo/bar
1470
1471 go 1.12
1472 -- a.go --
1473 package a
1474
1475 var A = 5
1476 -- b.go --
1477 package a
1478
1479 var B = 5
1480 -- c/c.go --
1481 package c
1482
1483 var C = 5
1484 -- d/d.go --
1485 package c
1486
1487 var D = 5
1488 -- .gitignore --
1489 b.go
1490 c/`)))
1491 if err != nil {
1492 t.Fatal(err)
1493 }
1494 defer cleanup()
1495
1496 gitInit(t, tmpDir)
1497 gitCommit(t, tmpDir)
1498
1499 for _, tc := range []struct {
1500 desc string
1501 subdir string
1502 wantFiles []string
1503 }{
1504 {
1505 desc: "from root",
1506 subdir: "",
1507 wantFiles: []string{"go.mod", "a.go", "d/d.go", ".gitignore"},
1508 },
1509 {
1510 desc: "from subdir",
1511 subdir: "d/",
1512 // Note: File paths are zipped as if the subdir were the root. ie d.go instead of d/d.go.
1513 wantFiles: []string{"d.go"},
1514 },
1515 } {
1516 t.Run(tc.desc, func(t *testing.T) {
1517 // Create zip from the directory.
1518 tmpZip := &bytes.Buffer{}
1519
1520 m := module.Version{Path: "example.com/foo/bar", Version: "v0.0.1"}
1521
1522 if err := modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", tc.subdir); err != nil {
1523 t.Fatal(err)
1524 }
1525
1526 readerAt := bytes.NewReader(tmpZip.Bytes())
1527 r, err := zip.NewReader(readerAt, int64(tmpZip.Len()))
1528 if err != nil {
1529 t.Fatal(err)
1530 }
1531 var gotFiles []string
1532 gotMap := map[string]bool{}
1533 for _, f := range r.File {
1534 gotMap[f.Name] = true
1535 gotFiles = append(gotFiles, f.Name)
1536 }
1537 wantMap := map[string]bool{}
1538 for _, f := range tc.wantFiles {
1539 p := filepath.Join("example.com", "foo", "bar@v0.0.1", f)
1540 wantMap[p] = true
1541 }
1542
1543 // The things that should be there.
1544 for f := range gotMap {
1545 if !wantMap[f] {
1546 t.Errorf("CreatedFromVCS: zipped file contains %s, but expected it not to", f)
1547 }
1548 }
1549
1550 // The things that are missing.
1551 for f := range wantMap {
1552 if !gotMap[f] {
1553 t.Errorf("CreatedFromVCS: zipped file doesn't contain %s, but expected it to. all files: %v", f, gotFiles)
1554 }
1555 }
1556 })
1557 }
1558 }
1559
1560 // Test what the experience of creating a zip from a v2 module is like.
1561 func TestCreateFromVCS_v2(t *testing.T) {
1562 // Write files to a temporary directory.
1563 tmpDir, cleanup, err := extractTxtarToTempDir(txtar.Parse([]byte(`-- go.mod --
1564 module example.com/foo/bar
1565
1566 go 1.12
1567 -- a.go --
1568 package a
1569
1570 var A = 5
1571 -- b.go --
1572 package a
1573
1574 var B = 5
1575 -- go.mod --
1576 module example.com/foo/bar
1577
1578 go 1.12
1579 -- gaz/v2/a_2.go --
1580 package a
1581
1582 var C = 9
1583 -- gaz/v2/b_2.go --
1584 package a
1585
1586 var B = 11
1587 -- gaz/v2/go.mod --
1588 module example.com/foo/bar/v2
1589
1590 go 1.12
1591 -- .gitignore --
1592 `)))
1593 if err != nil {
1594 t.Fatal(err)
1595 }
1596 defer cleanup()
1597
1598 gitInit(t, tmpDir)
1599 gitCommit(t, tmpDir)
1600
1601 // Create zip from the directory.
1602 tmpZip := &bytes.Buffer{}
1603
1604 m := module.Version{Path: "example.com/foo/bar/v2", Version: "v2.0.0"}
1605
1606 if err := modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", "gaz/v2"); err != nil {
1607 t.Fatal(err)
1608 }
1609
1610 readerAt := bytes.NewReader(tmpZip.Bytes())
1611 r, err := zip.NewReader(readerAt, int64(tmpZip.Len()))
1612 if err != nil {
1613 t.Fatal(err)
1614 }
1615 var gotFiles []string
1616 gotMap := map[string]bool{}
1617 for _, f := range r.File {
1618 gotMap[f.Name] = true
1619 gotFiles = append(gotFiles, f.Name)
1620 }
1621 wantMap := map[string]bool{
1622 "example.com/foo/bar/v2@v2.0.0/a_2.go": true,
1623 "example.com/foo/bar/v2@v2.0.0/b_2.go": true,
1624 "example.com/foo/bar/v2@v2.0.0/go.mod": true,
1625 }
1626
1627 // The things that should be there.
1628 for f := range gotMap {
1629 if !wantMap[f] {
1630 t.Errorf("CreatedFromVCS: zipped file contains %s, but expected it not to", f)
1631 }
1632 }
1633
1634 // The things that are missing.
1635 for f := range wantMap {
1636 if !gotMap[f] {
1637 t.Errorf("CreatedFromVCS: zipped file doesn't contain %s, but expected it to. all files: %v", f, gotFiles)
1638 }
1639 }
1640 }
1641
1642 func TestCreateFromVCS_nonGitDir(t *testing.T) {
1643 // Write files to a temporary directory.
1644 tmpDir, cleanup, err := extractTxtarToTempDir(txtar.Parse([]byte(`-- go.mod --
1645 module example.com/foo/bar
1646
1647 go 1.12
1648 -- a.go --
1649 package a
1650
1651 var A = 5
1652 `)))
1653 if err != nil {
1654 t.Fatal(err)
1655 }
1656 defer cleanup()
1657
1658 // Create zip from the directory.
1659 tmpZip, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
1660 if err != nil {
1661 t.Fatal(err)
1662 }
1663 tmpZipPath := tmpZip.Name()
1664 defer func() {
1665 tmpZip.Close()
1666 os.Remove(tmpZipPath)
1667 }()
1668
1669 m := module.Version{Path: "example.com/foo/bar", Version: "v0.0.1"}
1670
1671 if err := modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", ""); err == nil {
1672 t.Error("CreateFromVCS: expected error, got nil")
1673 }
1674 }
1675
1676 func TestCreateFromVCS_zeroCommitsGitDir(t *testing.T) {
1677 // Write files to a temporary directory.
1678 tmpDir, cleanup, err := extractTxtarToTempDir(txtar.Parse([]byte(`-- go.mod --
1679 module example.com/foo/bar
1680
1681 go 1.12
1682 -- a.go --
1683 package a
1684
1685 var A = 5
1686 `)))
1687 if err != nil {
1688 t.Fatal(err)
1689 }
1690 defer cleanup()
1691
1692 gitInit(t, tmpDir)
1693
1694 // Create zip from the directory.
1695 tmpZip, err := ioutil.TempFile("", "TestCreateFromDir-*.zip")
1696 if err != nil {
1697 t.Fatal(err)
1698 }
1699 tmpZipPath := tmpZip.Name()
1700 defer func() {
1701 tmpZip.Close()
1702 os.Remove(tmpZipPath)
1703 }()
1704
1705 m := module.Version{Path: "example.com/foo/bar", Version: "v0.0.1"}
1706
1707 if err := modzip.CreateFromVCS(tmpZip, m, tmpDir, "HEAD", ""); err == nil {
1708 t.Error("CreateFromVCS: expected error, got nil")
1709 }
1710 }
1711
1712 // gitInit runs "git init" at the specified dir.
1713 //
1714 // Note: some environments - and trybots - don't have git installed. This
1715 // function will cause the calling test to be skipped if that's the case.
1716 func gitInit(t *testing.T, dir string) {
1717 t.Helper()
1718
1719 if _, err := exec.LookPath("git"); err != nil {
1720 t.Skip("PATH does not contain git")
1721 }
1722
1723 cmd := exec.Command("git", "init")
1724 cmd.Dir = dir
1725 cmd.Stderr = os.Stderr
1726 if err := cmd.Run(); err != nil {
1727 t.Fatal(err)
1728 }
1729
1730 cmd = exec.Command("git", "config", "user.email", "testing@golangtests.com")
1731 cmd.Dir = dir
1732 cmd.Stderr = os.Stderr
1733 if err := cmd.Run(); err != nil {
1734 t.Fatal(err)
1735 }
1736
1737 cmd = exec.Command("git", "config", "user.name", "This is the zip Go tests")
1738 cmd.Dir = dir
1739 cmd.Stderr = os.Stderr
1740 if err := cmd.Run(); err != nil {
1741 t.Fatal(err)
1742 }
1743 }
1744
1745 func gitCommit(t *testing.T, dir string) {
1746 t.Helper()
1747
1748 if _, err := exec.LookPath("git"); err != nil {
1749 t.Skip("PATH does not contain git")
1750 }
1751
1752 cmd := exec.Command("git", "add", "-A")
1753 cmd.Dir = dir
1754 cmd.Stderr = os.Stderr
1755 if err := cmd.Run(); err != nil {
1756 t.Skip("git executable is not available on this machine")
1757 }
1758
1759 cmd = exec.Command("git", "commit", "-m", "some commit")
1760 cmd.Dir = dir
1761 cmd.Stderr = os.Stderr
1762 if err := cmd.Run(); err != nil {
1763 t.Fatal(err)
1764 }
1765 }