diff options
| author | 2023-04-28 01:36:55 +0800 | |
|---|---|---|
| committer | 2023-04-28 01:36:55 +0800 | |
| commit | fc8c5fdce62fb229202659408798a7b6c98f6e8b (patch) | |
| tree | 7554f80e50de4af6fd255afa7c21bcdd58a7af34 /cli/internal/fs | |
| parent | dd84b9d64fb98746a230cd24233ff50a562c39c9 (diff) | |
| download | HydroRoll-fc8c5fdce62fb229202659408798a7b6c98f6e8b.tar.gz HydroRoll-fc8c5fdce62fb229202659408798a7b6c98f6e8b.zip | |
Diffstat (limited to 'cli/internal/fs')
23 files changed, 0 insertions, 2357 deletions
diff --git a/cli/internal/fs/copy_file.go b/cli/internal/fs/copy_file.go deleted file mode 100644 index e7619de..0000000 --- a/cli/internal/fs/copy_file.go +++ /dev/null @@ -1,81 +0,0 @@ -// Adapted from https://github.com/thought-machine/please -// Copyright Thought Machine, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -package fs - -import ( - "errors" - "os" - "path/filepath" - - "github.com/karrick/godirwalk" -) - -// RecursiveCopy copies either a single file or a directory. -// 'mode' is the mode of the destination file. -func RecursiveCopy(from string, to string) error { - // Verified all callers are passing in absolute paths for from (and to) - statedFrom := LstatCachedFile{Path: UnsafeToAbsoluteSystemPath(from)} - fromType, err := statedFrom.GetType() - if err != nil { - return err - } - - if fromType.IsDir() { - return WalkMode(statedFrom.Path.ToStringDuringMigration(), func(name string, isDir bool, fileType os.FileMode) error { - dest := filepath.Join(to, name[len(statedFrom.Path.ToString()):]) - // name is absolute, (originates from godirwalk) - src := LstatCachedFile{Path: UnsafeToAbsoluteSystemPath(name), fileType: &fileType} - if isDir { - mode, err := src.GetMode() - if err != nil { - return err - } - return os.MkdirAll(dest, mode) - } - return CopyFile(&src, dest) - }) - } - return CopyFile(&statedFrom, to) -} - -// Walk implements an equivalent to filepath.Walk. -// It's implemented over github.com/karrick/godirwalk but the provided interface doesn't use that -// to make it a little easier to handle. -func Walk(rootPath string, callback func(name string, isDir bool) error) error { - return WalkMode(rootPath, func(name string, isDir bool, mode os.FileMode) error { - return callback(name, isDir) - }) -} - -// WalkMode is like Walk but the callback receives an additional type specifying the file mode type. -// N.B. This only includes the bits of the mode that determine the mode type, not the permissions. -func WalkMode(rootPath string, callback func(name string, isDir bool, mode os.FileMode) error) error { - return godirwalk.Walk(rootPath, &godirwalk.Options{ - Callback: func(name string, info *godirwalk.Dirent) error { - // currently we support symlinked files, but not symlinked directories: - // For copying, we Mkdir and bail if we encounter a symlink to a directoy - // For finding packages, we enumerate the symlink, but don't follow inside - isDir, err := info.IsDirOrSymlinkToDir() - if err != nil { - pathErr := &os.PathError{} - if errors.As(err, &pathErr) { - // If we have a broken link, skip this entry - return godirwalk.SkipThis - } - return err - } - return callback(name, isDir, info.ModeType()) - }, - ErrorCallback: func(pathname string, err error) godirwalk.ErrorAction { - pathErr := &os.PathError{} - if errors.As(err, &pathErr) { - return godirwalk.SkipNode - } - return godirwalk.Halt - }, - Unsorted: true, - AllowNonDirectory: true, - FollowSymbolicLinks: false, - }) -} diff --git a/cli/internal/fs/copy_file_test.go b/cli/internal/fs/copy_file_test.go deleted file mode 100644 index 6a61576..0000000 --- a/cli/internal/fs/copy_file_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package fs - -import ( - "errors" - "io/ioutil" - "os" - "path/filepath" - "testing" - - "github.com/vercel/turbo/cli/internal/turbopath" - "gotest.tools/v3/assert" - "gotest.tools/v3/fs" -) - -func TestCopyFile(t *testing.T) { - srcTmpDir := turbopath.AbsoluteSystemPath(t.TempDir()) - destTmpDir := turbopath.AbsoluteSystemPath(t.TempDir()) - srcFilePath := srcTmpDir.UntypedJoin("src") - destFilePath := destTmpDir.UntypedJoin("dest") - from := &LstatCachedFile{Path: srcFilePath} - - // The src file doesn't exist, will error. - err := CopyFile(from, destFilePath.ToString()) - pathErr := &os.PathError{} - if !errors.As(err, &pathErr) { - t.Errorf("got %v, want PathError", err) - } - - // Create the src file. - srcFile, err := srcFilePath.Create() - assert.NilError(t, err, "Create") - _, err = srcFile.WriteString("src") - assert.NilError(t, err, "WriteString") - assert.NilError(t, srcFile.Close(), "Close") - - // Copy the src to the dest. - err = CopyFile(from, destFilePath.ToString()) - assert.NilError(t, err, "src exists dest does not, should not error.") - - // Now test for symlinks. - symlinkSrcDir := turbopath.AbsoluteSystemPath(t.TempDir()) - symlinkTargetDir := turbopath.AbsoluteSystemPath(t.TempDir()) - symlinkDestDir := turbopath.AbsoluteSystemPath(t.TempDir()) - symlinkSrcPath := symlinkSrcDir.UntypedJoin("symlink") - symlinkTargetPath := symlinkTargetDir.UntypedJoin("target") - symlinkDestPath := symlinkDestDir.UntypedJoin("dest") - fromSymlink := &LstatCachedFile{Path: symlinkSrcPath} - - // Create the symlink target. - symlinkTargetFile, err := symlinkTargetPath.Create() - assert.NilError(t, err, "Create") - _, err = symlinkTargetFile.WriteString("Target") - assert.NilError(t, err, "WriteString") - assert.NilError(t, symlinkTargetFile.Close(), "Close") - - // Link things up. - err = symlinkSrcPath.Symlink(symlinkTargetPath.ToString()) - assert.NilError(t, err, "Symlink") - - // Run the test. - err = CopyFile(fromSymlink, symlinkDestPath.ToString()) - assert.NilError(t, err, "Copying a valid symlink does not error.") - - // Break the symlink. - err = symlinkTargetPath.Remove() - assert.NilError(t, err, "breaking the symlink") - - // Remove the existing copy. - err = symlinkDestPath.Remove() - assert.NilError(t, err, "existing copy is removed") - - // Try copying the now-broken symlink. - err = CopyFile(fromSymlink, symlinkDestPath.ToString()) - assert.NilError(t, err, "CopyFile") - - // Confirm that it copied - target, err := symlinkDestPath.Readlink() - assert.NilError(t, err, "Readlink") - assert.Equal(t, target, symlinkTargetPath.ToString()) -} - -func TestCopyOrLinkFileWithPerms(t *testing.T) { - // Directory layout: - // - // <src>/ - // foo - readonlyMode := os.FileMode(0444) - srcDir := turbopath.AbsoluteSystemPath(t.TempDir()) - dstDir := turbopath.AbsoluteSystemPath(t.TempDir()) - srcFilePath := srcDir.UntypedJoin("src") - dstFilePath := dstDir.UntypedJoin("dst") - srcFile, err := srcFilePath.Create() - defer func() { _ = srcFile.Close() }() - assert.NilError(t, err, "Create") - err = srcFile.Chmod(readonlyMode) - assert.NilError(t, err, "Chmod") - err = CopyFile(&LstatCachedFile{Path: srcFilePath}, dstFilePath.ToStringDuringMigration()) - assert.NilError(t, err, "CopyOrLinkFile") - info, err := dstFilePath.Lstat() - assert.NilError(t, err, "Lstat") - assert.Equal(t, info.Mode(), readonlyMode, "expected dest to have matching permissions") -} - -func TestRecursiveCopy(t *testing.T) { - // Directory layout: - // - // <src>/ - // b - // child/ - // a - // link -> ../b - // broken -> missing - // circle -> ../child - src := fs.NewDir(t, "recursive-copy-or-link") - dst := fs.NewDir(t, "recursive-copy-or-link-dist") - childDir := filepath.Join(src.Path(), "child") - err := os.Mkdir(childDir, os.ModeDir|0777) - assert.NilError(t, err, "Mkdir") - aPath := filepath.Join(childDir, "a") - aFile, err := os.Create(aPath) - assert.NilError(t, err, "Create") - _, err = aFile.WriteString("hello") - assert.NilError(t, err, "WriteString") - assert.NilError(t, aFile.Close(), "Close") - - bPath := filepath.Join(src.Path(), "b") - bFile, err := os.Create(bPath) - assert.NilError(t, err, "Create") - _, err = bFile.WriteString("bFile") - assert.NilError(t, err, "WriteString") - assert.NilError(t, bFile.Close(), "Close") - - srcLinkPath := filepath.Join(childDir, "link") - assert.NilError(t, os.Symlink(filepath.FromSlash("../b"), srcLinkPath), "Symlink") - - srcBrokenLinkPath := filepath.Join(childDir, "broken") - assert.NilError(t, os.Symlink("missing", srcBrokenLinkPath), "Symlink") - circlePath := filepath.Join(childDir, "circle") - assert.NilError(t, os.Symlink(filepath.FromSlash("../child"), circlePath), "Symlink") - - err = RecursiveCopy(src.Path(), dst.Path()) - assert.NilError(t, err, "RecursiveCopy") - // For ensure multiple times copy will not broken - err = RecursiveCopy(src.Path(), dst.Path()) - assert.NilError(t, err, "RecursiveCopy") - - dstChildDir := filepath.Join(dst.Path(), "child") - assertDirMatches(t, childDir, dstChildDir) - dstAPath := filepath.Join(dst.Path(), "child", "a") - assertFileMatches(t, aPath, dstAPath) - dstBPath := filepath.Join(dst.Path(), "b") - assertFileMatches(t, bPath, dstBPath) - dstLinkPath := filepath.Join(dst.Path(), "child", "link") - dstLinkDest, err := os.Readlink(dstLinkPath) - assert.NilError(t, err, "Readlink") - expectedLinkDest := filepath.FromSlash("../b") - if dstLinkDest != expectedLinkDest { - t.Errorf("Readlink got %v, want %v", dstLinkDest, expectedLinkDest) - } - dstBrokenLinkPath := filepath.Join(dst.Path(), "child", "broken") - brokenLinkExists := PathExists(dstBrokenLinkPath) - if brokenLinkExists { - t.Errorf("We cached a broken link at %v", dstBrokenLinkPath) - } - // Currently, we convert symlink-to-directory to empty-directory - // This is very likely not ideal behavior, but leaving this test here to verify - // that it is what we expect at this point in time. - dstCirclePath := filepath.Join(dst.Path(), "child", "circle") - circleStat, err := os.Lstat(dstCirclePath) - assert.NilError(t, err, "Lstat") - assert.Equal(t, circleStat.IsDir(), true) - entries, err := os.ReadDir(dstCirclePath) - assert.NilError(t, err, "ReadDir") - assert.Equal(t, len(entries), 0) -} - -func assertFileMatches(t *testing.T, orig string, copy string) { - t.Helper() - origBytes, err := ioutil.ReadFile(orig) - assert.NilError(t, err, "ReadFile") - copyBytes, err := ioutil.ReadFile(copy) - assert.NilError(t, err, "ReadFile") - assert.DeepEqual(t, origBytes, copyBytes) - origStat, err := os.Lstat(orig) - assert.NilError(t, err, "Lstat") - copyStat, err := os.Lstat(copy) - assert.NilError(t, err, "Lstat") - assert.Equal(t, origStat.Mode(), copyStat.Mode()) -} - -func assertDirMatches(t *testing.T, orig string, copy string) { - t.Helper() - origStat, err := os.Lstat(orig) - assert.NilError(t, err, "Lstat") - copyStat, err := os.Lstat(copy) - assert.NilError(t, err, "Lstat") - assert.Equal(t, origStat.Mode(), copyStat.Mode()) -} diff --git a/cli/internal/fs/fs.go b/cli/internal/fs/fs.go deleted file mode 100644 index 77804c0..0000000 --- a/cli/internal/fs/fs.go +++ /dev/null @@ -1,191 +0,0 @@ -package fs - -import ( - "io" - "io/ioutil" - "log" - "os" - "path/filepath" - "runtime" - "strings" - - "github.com/pkg/errors" - "github.com/vercel/turbo/cli/internal/util" -) - -// https://github.com/thought-machine/please/blob/master/src/fs/fs.go - -// DirPermissions are the default permission bits we apply to directories. -const DirPermissions = os.ModeDir | 0775 - -// EnsureDir ensures that the directory of the given file has been created. -func EnsureDir(filename string) error { - dir := filepath.Dir(filename) - err := os.MkdirAll(dir, DirPermissions) - if err != nil && FileExists(dir) { - // It looks like this is a file and not a directory. Attempt to remove it; this can - // happen in some cases if you change a rule from outputting a file to a directory. - log.Printf("Attempting to remove file %s; a subdirectory is required", dir) - if err2 := os.Remove(dir); err2 == nil { - err = os.MkdirAll(dir, DirPermissions) - } else { - return err - } - } - return err -} - -var nonRelativeSentinel string = ".." + string(filepath.Separator) - -// DirContainsPath returns true if the path 'target' is contained within 'dir' -// Expects both paths to be absolute and does not verify that either path exists. -func DirContainsPath(dir string, target string) (bool, error) { - // On windows, trying to get a relative path between files on different volumes - // is an error. We don't care about the error, it's good enough for us to say - // that one path doesn't contain the other if they're on different volumes. - if runtime.GOOS == "windows" && filepath.VolumeName(dir) != filepath.VolumeName(target) { - return false, nil - } - // In Go, filepath.Rel can return a path that starts with "../" or equivalent. - // Checking filesystem-level contains can get extremely complicated - // (see https://github.com/golang/dep/blob/f13583b555deaa6742f141a9c1185af947720d60/internal/fs/fs.go#L33) - // As a compromise, rely on the stdlib to generate a relative path and then check - // if the first step is "../". - rel, err := filepath.Rel(dir, target) - if err != nil { - return false, err - } - return !strings.HasPrefix(rel, nonRelativeSentinel), nil -} - -// PathExists returns true if the given path exists, as a file or a directory. -func PathExists(filename string) bool { - _, err := os.Lstat(filename) - return err == nil -} - -// FileExists returns true if the given path exists and is a file. -func FileExists(filename string) bool { - info, err := os.Lstat(filename) - return err == nil && !info.IsDir() -} - -// CopyFile copies a file from 'from' to 'to', with an attempt to perform a copy & rename -// to avoid chaos if anything goes wrong partway. -func CopyFile(from *LstatCachedFile, to string) error { - fromMode, err := from.GetMode() - if err != nil { - return errors.Wrapf(err, "getting mode for %v", from.Path) - } - if fromMode&os.ModeSymlink != 0 { - target, err := from.Path.Readlink() - if err != nil { - return errors.Wrapf(err, "reading link target for %v", from.Path) - } - if err := EnsureDir(to); err != nil { - return err - } - if _, err := os.Lstat(to); err == nil { - // target link file exist, should remove it first - err := os.Remove(to) - if err != nil { - return err - } - } - return os.Symlink(target, to) - } - fromFile, err := from.Path.Open() - if err != nil { - return err - } - defer util.CloseAndIgnoreError(fromFile) - return writeFileFromStream(fromFile, to, fromMode) -} - -// writeFileFromStream writes data from a reader to the file named 'to', with an attempt to perform -// a copy & rename to avoid chaos if anything goes wrong partway. -func writeFileFromStream(fromFile io.Reader, to string, mode os.FileMode) error { - dir, file := filepath.Split(to) - if dir != "" { - if err := os.MkdirAll(dir, DirPermissions); err != nil { - return err - } - } - tempFile, err := ioutil.TempFile(dir, file) - if err != nil { - return err - } - if _, err := io.Copy(tempFile, fromFile); err != nil { - return err - } - if err := tempFile.Close(); err != nil { - return err - } - // OK, now file is written; adjust permissions appropriately. - if mode == 0 { - mode = 0664 - } - if err := os.Chmod(tempFile.Name(), mode); err != nil { - return err - } - // And move it to its final destination. - return renameFile(tempFile.Name(), to) -} - -// IsDirectory checks if a given path is a directory -func IsDirectory(path string) bool { - info, err := os.Stat(path) - return err == nil && info.IsDir() -} - -// Try to gracefully rename the file as the os.Rename does not work across -// filesystems and on most Linux systems /tmp is mounted as tmpfs -func renameFile(from, to string) (err error) { - err = os.Rename(from, to) - if err == nil { - return nil - } - err = copyFile(from, to) - if err != nil { - return err - } - err = os.RemoveAll(from) - if err != nil { - return err - } - return nil -} - -func copyFile(from, to string) (err error) { - in, err := os.Open(from) - if err != nil { - return err - } - defer in.Close() - - out, err := os.Create(to) - if err != nil { - return err - } - defer func() { - if e := out.Close(); e != nil { - err = e - } - }() - - _, err = io.Copy(out, in) - if err != nil { - return err - } - - si, err := os.Stat(from) - if err != nil { - return err - } - err = os.Chmod(to, si.Mode()) - if err != nil { - return err - } - - return nil -} diff --git a/cli/internal/fs/fs_test.go b/cli/internal/fs/fs_test.go deleted file mode 100644 index 0598d43..0000000 --- a/cli/internal/fs/fs_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package fs - -import ( - "path/filepath" - "testing" -) - -func Test_DirContainsPath(t *testing.T) { - parent, err := filepath.Abs(filepath.Join("some", "path")) - if err != nil { - t.Fatalf("failed to construct parent path %v", err) - } - testcases := []struct { - target []string - want bool - }{ - { - []string{"..", "elsewhere"}, - false, - }, - { - []string{"sibling"}, - false, - }, - { - // The same path as parent - []string{"some", "path"}, - true, - }, - { - []string{"some", "path", "..", "path", "inside", "parent"}, - true, - }, - { - []string{"some", "path", "inside", "..", "inside", "parent"}, - true, - }, - { - []string{"some", "path", "inside", "..", "..", "outside", "parent"}, - false, - }, - { - []string{"some", "pathprefix"}, - false, - }, - } - for _, tc := range testcases { - target, err := filepath.Abs(filepath.Join(tc.target...)) - if err != nil { - t.Fatalf("failed to construct path for %v: %v", tc.target, err) - } - got, err := DirContainsPath(parent, target) - if err != nil { - t.Fatalf("failed to check ") - } - if got != tc.want { - t.Errorf("DirContainsPath(%v, %v) got %v, want %v", parent, target, got, tc.want) - } - } -} diff --git a/cli/internal/fs/fs_windows_test.go b/cli/internal/fs/fs_windows_test.go deleted file mode 100644 index 4e71e2c..0000000 --- a/cli/internal/fs/fs_windows_test.go +++ /dev/null @@ -1,18 +0,0 @@ -//go:build windows -// +build windows - -package fs - -import "testing" - -func TestDifferentVolumes(t *testing.T) { - p1 := "C:\\some\\path" - p2 := "D:\\other\\path" - contains, err := DirContainsPath(p1, p2) - if err != nil { - t.Errorf("DirContainsPath got error %v, want <nil>", err) - } - if contains { - t.Errorf("DirContainsPath got true, want false") - } -} diff --git a/cli/internal/fs/get_turbo_data_dir_go.go b/cli/internal/fs/get_turbo_data_dir_go.go deleted file mode 100644 index 2cf459a..0000000 --- a/cli/internal/fs/get_turbo_data_dir_go.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build go || !rust -// +build go !rust - -package fs - -import ( - "github.com/adrg/xdg" - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// GetTurboDataDir returns a directory outside of the repo -// where turbo can store data files related to turbo. -func GetTurboDataDir() turbopath.AbsoluteSystemPath { - dataHome := AbsoluteSystemPathFromUpstream(xdg.DataHome) - return dataHome.UntypedJoin("turborepo") -} diff --git a/cli/internal/fs/get_turbo_data_dir_rust.go b/cli/internal/fs/get_turbo_data_dir_rust.go deleted file mode 100644 index dbc80f3..0000000 --- a/cli/internal/fs/get_turbo_data_dir_rust.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build rust -// +build rust - -package fs - -import ( - "github.com/vercel/turbo/cli/internal/ffi" - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// GetTurboDataDir returns a directory outside of the repo -// where turbo can store data files related to turbo. -func GetTurboDataDir() turbopath.AbsoluteSystemPath { - dir := ffi.GetTurboDataDir() - return turbopath.AbsoluteSystemPathFromUpstream(dir) -} diff --git a/cli/internal/fs/hash.go b/cli/internal/fs/hash.go deleted file mode 100644 index fed7d87..0000000 --- a/cli/internal/fs/hash.go +++ /dev/null @@ -1,61 +0,0 @@ -package fs - -import ( - "crypto/sha1" - "encoding/hex" - "fmt" - "io" - "os" - "strconv" - - "github.com/vercel/turbo/cli/internal/xxhash" -) - -func HashObject(i interface{}) (string, error) { - hash := xxhash.New() - - _, err := hash.Write([]byte(fmt.Sprintf("%v", i))) - - return hex.EncodeToString(hash.Sum(nil)), err -} - -func HashFile(filePath string) (string, error) { - file, err := os.Open(filePath) - if err != nil { - return "", err - } - defer file.Close() - - hash := xxhash.New() - if _, err := io.Copy(hash, file); err != nil { - return "", err - } - - return hex.EncodeToString(hash.Sum(nil)), nil -} - -// GitLikeHashFile is a function that mimics how Git -// calculates the SHA1 for a file (or, in Git terms, a "blob") (without git) -func GitLikeHashFile(filePath string) (string, error) { - file, err := os.Open(filePath) - if err != nil { - return "", err - } - defer file.Close() - - stat, err := file.Stat() - if err != nil { - return "", err - } - hash := sha1.New() - hash.Write([]byte("blob")) - hash.Write([]byte(" ")) - hash.Write([]byte(strconv.FormatInt(stat.Size(), 10))) - hash.Write([]byte{0}) - - if _, err := io.Copy(hash, file); err != nil { - return "", err - } - - return hex.EncodeToString(hash.Sum(nil)), nil -} diff --git a/cli/internal/fs/hash_test.go b/cli/internal/fs/hash_test.go deleted file mode 100644 index dd2fa84..0000000 --- a/cli/internal/fs/hash_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package fs - -import ( - "testing" - - "gotest.tools/v3/assert" -) - -const _numOfRuns = 20 - -func Test_HashObjectStability(t *testing.T) { - type TestCase struct { - name string - obj interface{} - } - type complexStruct struct { - nested TaskOutputs - foo string - bar []string - } - - testCases := []TestCase{ - { - name: "task object", - obj: TaskOutputs{ - Inclusions: []string{"foo", "bar"}, - Exclusions: []string{"baz"}, - }, - }, - { - name: "complex struct", - obj: complexStruct{ - nested: TaskOutputs{ - Exclusions: []string{"bar", "baz"}, - Inclusions: []string{"foo"}, - }, - foo: "a", - bar: []string{"b", "c"}, - }, - }, - } - - for _, tc := range testCases { - expectedHash, err := HashObject(tc.obj) - assert.NilError(t, err, tc.name) - - for n := 0; n < _numOfRuns; n++ { - hash, err := HashObject(tc.obj) - assert.NilError(t, err, tc.name) - assert.Equal(t, expectedHash, hash, tc.name) - } - } -} diff --git a/cli/internal/fs/lstat.go b/cli/internal/fs/lstat.go deleted file mode 100644 index eff0810..0000000 --- a/cli/internal/fs/lstat.go +++ /dev/null @@ -1,74 +0,0 @@ -package fs - -import ( - "io/fs" - "os" - - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// LstatCachedFile maintains a cache of file info, mode and type for the given Path -type LstatCachedFile struct { - Path turbopath.AbsoluteSystemPath - fileInfo fs.FileInfo - fileMode *fs.FileMode - fileType *fs.FileMode -} - -// GetInfo returns, and caches the file info for the LstatCachedFile.Path -func (file *LstatCachedFile) GetInfo() (fs.FileInfo, error) { - if file.fileInfo != nil { - return file.fileInfo, nil - } - - err := file.lstat() - if err != nil { - return nil, err - } - - return file.fileInfo, nil -} - -// GetMode returns, and caches the file mode for the LstatCachedFile.Path -func (file *LstatCachedFile) GetMode() (fs.FileMode, error) { - if file.fileMode != nil { - return *file.fileMode, nil - } - - err := file.lstat() - if err != nil { - return 0, err - } - - return *file.fileMode, nil -} - -// GetType returns, and caches the type bits of (FileMode & os.ModeType) for the LstatCachedFile.Path -func (file *LstatCachedFile) GetType() (fs.FileMode, error) { - if file.fileType != nil { - return *file.fileType, nil - } - - err := file.lstat() - if err != nil { - return 0, err - } - - return *file.fileType, nil -} - -func (file *LstatCachedFile) lstat() error { - fileInfo, err := file.Path.Lstat() - if err != nil { - return err - } - - fileMode := fileInfo.Mode() - fileModeType := fileMode & os.ModeType - - file.fileInfo = fileInfo - file.fileMode = &fileMode - file.fileType = &fileModeType - - return nil -} diff --git a/cli/internal/fs/package_json.go b/cli/internal/fs/package_json.go deleted file mode 100644 index 883f7a4..0000000 --- a/cli/internal/fs/package_json.go +++ /dev/null @@ -1,142 +0,0 @@ -package fs - -import ( - "bytes" - "encoding/json" - "sync" - - "github.com/vercel/turbo/cli/internal/lockfile" - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// PackageJSON represents NodeJS package.json -type PackageJSON struct { - Name string `json:"name"` - Version string `json:"version"` - Scripts map[string]string `json:"scripts"` - Dependencies map[string]string `json:"dependencies"` - DevDependencies map[string]string `json:"devDependencies"` - OptionalDependencies map[string]string `json:"optionalDependencies"` - PeerDependencies map[string]string `json:"peerDependencies"` - PackageManager string `json:"packageManager"` - Os []string `json:"os"` - Workspaces Workspaces `json:"workspaces"` - Private bool `json:"private"` - // Exact JSON object stored in package.json including unknown fields - // During marshalling struct fields will take priority over raw fields - RawJSON map[string]interface{} `json:"-"` - - // relative path from repo root to the package.json file - PackageJSONPath turbopath.AnchoredSystemPath `json:"-"` - // relative path from repo root to the package - Dir turbopath.AnchoredSystemPath `json:"-"` - InternalDeps []string `json:"-"` - UnresolvedExternalDeps map[string]string `json:"-"` - TransitiveDeps []lockfile.Package `json:"-"` - LegacyTurboConfig *TurboJSON `json:"turbo"` - Mu sync.Mutex `json:"-"` - ExternalDepsHash string `json:"-"` -} - -type Workspaces []string - -type WorkspacesAlt struct { - Packages []string `json:"packages,omitempty"` -} - -func (r *Workspaces) UnmarshalJSON(data []byte) error { - var tmp = &WorkspacesAlt{} - if err := json.Unmarshal(data, tmp); err == nil { - *r = Workspaces(tmp.Packages) - return nil - } - var tempstr = []string{} - if err := json.Unmarshal(data, &tempstr); err != nil { - return err - } - *r = tempstr - return nil -} - -// ReadPackageJSON returns a struct of package.json -func ReadPackageJSON(path turbopath.AbsoluteSystemPath) (*PackageJSON, error) { - b, err := path.ReadFile() - if err != nil { - return nil, err - } - return UnmarshalPackageJSON(b) -} - -// UnmarshalPackageJSON decodes a byte slice into a PackageJSON struct -func UnmarshalPackageJSON(data []byte) (*PackageJSON, error) { - var rawJSON map[string]interface{} - if err := json.Unmarshal(data, &rawJSON); err != nil { - return nil, err - } - - pkgJSON := &PackageJSON{} - if err := json.Unmarshal(data, &pkgJSON); err != nil { - return nil, err - } - pkgJSON.RawJSON = rawJSON - - return pkgJSON, nil -} - -// MarshalPackageJSON Serialize PackageJSON to a slice of bytes -func MarshalPackageJSON(pkgJSON *PackageJSON) ([]byte, error) { - structuredContent, err := json.Marshal(pkgJSON) - if err != nil { - return nil, err - } - var structuredFields map[string]interface{} - if err := json.Unmarshal(structuredContent, &structuredFields); err != nil { - return nil, err - } - - fieldsToSerialize := make(map[string]interface{}, len(pkgJSON.RawJSON)) - - // copy pkgJSON.RawJSON - for key, value := range pkgJSON.RawJSON { - fieldsToSerialize[key] = value - } - - for key, value := range structuredFields { - if isEmpty(value) { - delete(fieldsToSerialize, key) - } else { - fieldsToSerialize[key] = value - } - } - - var b bytes.Buffer - encoder := json.NewEncoder(&b) - encoder.SetEscapeHTML(false) - encoder.SetIndent("", " ") - if err := encoder.Encode(fieldsToSerialize); err != nil { - return nil, err - } - - return b.Bytes(), nil -} - -func isEmpty(value interface{}) bool { - if value == nil { - return true - } - switch s := value.(type) { - case string: - return s == "" - case bool: - return !s - case []string: - return len(s) == 0 - case map[string]interface{}: - return len(s) == 0 - case Workspaces: - return len(s) == 0 - default: - // Assume any unknown types aren't empty - return false - } -} diff --git a/cli/internal/fs/package_json_test.go b/cli/internal/fs/package_json_test.go deleted file mode 100644 index 3c16620..0000000 --- a/cli/internal/fs/package_json_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package fs - -import ( - "testing" - - "gotest.tools/v3/assert" -) - -func Test_UnmarshalPackageJSON(t *testing.T) { - type Case struct { - name string - json string - expectedFields *PackageJSON - } - - testCases := []Case{ - { - name: "basic types are in raw and processed", - json: `{"name":"foo","version":"1.2.3"}`, - expectedFields: &PackageJSON{ - Name: "foo", - Version: "1.2.3", - RawJSON: map[string]interface{}{ - "name": "foo", - "version": "1.2.3", - }, - }, - }, - { - name: "map types get copied", - json: `{"dependencies":{"foo":"1.2.3"},"devDependencies":{"bar": "^1.0.0"}}`, - expectedFields: &PackageJSON{ - Dependencies: map[string]string{"foo": "1.2.3"}, - DevDependencies: map[string]string{"bar": "^1.0.0"}, - RawJSON: map[string]interface{}{ - "dependencies": map[string]interface{}{"foo": "1.2.3"}, - "devDependencies": map[string]interface{}{"bar": "^1.0.0"}, - }, - }, - }, - { - name: "array types get copied", - json: `{"os":["linux", "windows"]}`, - expectedFields: &PackageJSON{ - Os: []string{"linux", "windows"}, - RawJSON: map[string]interface{}{ - "os": []interface{}{"linux", "windows"}, - }, - }, - }, - } - - for _, testCase := range testCases { - actual, err := UnmarshalPackageJSON([]byte(testCase.json)) - assert.NilError(t, err, testCase.name) - assertPackageJSONEqual(t, actual, testCase.expectedFields) - } -} - -func Test_MarshalPackageJSON(t *testing.T) { - type TestCase struct { - name string - input *PackageJSON - expected *PackageJSON - } - - testCases := []TestCase{ - { - name: "roundtrip should have no effect", - input: &PackageJSON{ - Name: "foo", - Version: "1.2.3", - RawJSON: map[string]interface{}{ - "name": "foo", - "version": "1.2.3", - }, - }, - expected: &PackageJSON{ - Name: "foo", - Version: "1.2.3", - RawJSON: map[string]interface{}{ - "name": "foo", - "version": "1.2.3", - }, - }, - }, - { - name: "structured fields should take priority over raw values", - input: &PackageJSON{ - Name: "foo", - Version: "2.3.4", - RawJSON: map[string]interface{}{ - "name": "foo", - "version": "1.2.3", - }, - }, - expected: &PackageJSON{ - Name: "foo", - Version: "2.3.4", - RawJSON: map[string]interface{}{ - "name": "foo", - "version": "2.3.4", - }, - }, - }, - { - name: "empty structured fields don't get serialized", - input: &PackageJSON{ - Name: "foo", - Version: "", - RawJSON: map[string]interface{}{ - "name": "foo", - "version": "1.2.3", - }, - }, - expected: &PackageJSON{ - Name: "foo", - Version: "", - RawJSON: map[string]interface{}{ - "name": "foo", - }, - }, - }, - { - name: "unstructured fields survive the round trip", - input: &PackageJSON{ - Name: "foo", - RawJSON: map[string]interface{}{ - "name": "foo", - "special-field": "special-value", - "special-config": map[string]interface{}{ - "flag": true, - "value": "toggled", - }, - }, - }, - expected: &PackageJSON{ - Name: "foo", - RawJSON: map[string]interface{}{ - "name": "foo", - "special-field": "special-value", - "special-config": map[string]interface{}{ - "flag": true, - "value": "toggled", - }, - }, - }, - }, - } - - for _, testCase := range testCases { - serializedInput, err := MarshalPackageJSON(testCase.input) - assert.NilError(t, err, testCase.name) - actual, err := UnmarshalPackageJSON(serializedInput) - assert.NilError(t, err, testCase.name) - assertPackageJSONEqual(t, actual, testCase.expected) - } -} - -// Asserts that the data section of two PackageJSON structs are equal -func assertPackageJSONEqual(t *testing.T, x *PackageJSON, y *PackageJSON) { - t.Helper() - assert.Equal(t, x.Name, y.Name) - assert.Equal(t, x.Version, y.Version) - assert.DeepEqual(t, x.Scripts, y.Scripts) - assert.DeepEqual(t, x.Dependencies, y.Dependencies) - assert.DeepEqual(t, x.DevDependencies, y.DevDependencies) - assert.DeepEqual(t, x.OptionalDependencies, y.OptionalDependencies) - assert.DeepEqual(t, x.PeerDependencies, y.PeerDependencies) - assert.Equal(t, x.PackageManager, y.PackageManager) - assert.DeepEqual(t, x.Workspaces, y.Workspaces) - assert.DeepEqual(t, x.Private, y.Private) - assert.DeepEqual(t, x.RawJSON, y.RawJSON) -} diff --git a/cli/internal/fs/path.go b/cli/internal/fs/path.go deleted file mode 100644 index 2023d69..0000000 --- a/cli/internal/fs/path.go +++ /dev/null @@ -1,113 +0,0 @@ -package fs - -import ( - "fmt" - iofs "io/fs" - "os" - "path/filepath" - "reflect" - - "github.com/adrg/xdg" - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// CheckedToAbsoluteSystemPath inspects a string and determines if it is an absolute path. -func CheckedToAbsoluteSystemPath(s string) (turbopath.AbsoluteSystemPath, error) { - if filepath.IsAbs(s) { - return turbopath.AbsoluteSystemPath(s), nil - } - return "", fmt.Errorf("%v is not an absolute path", s) -} - -// ResolveUnknownPath returns unknown if it is an absolute path, otherwise, it -// assumes unknown is a path relative to the given root. -func ResolveUnknownPath(root turbopath.AbsoluteSystemPath, unknown string) turbopath.AbsoluteSystemPath { - if filepath.IsAbs(unknown) { - return turbopath.AbsoluteSystemPath(unknown) - } - return root.UntypedJoin(unknown) -} - -// UnsafeToAbsoluteSystemPath directly converts a string to an AbsoluteSystemPath -func UnsafeToAbsoluteSystemPath(s string) turbopath.AbsoluteSystemPath { - return turbopath.AbsoluteSystemPath(s) -} - -// UnsafeToAnchoredSystemPath directly converts a string to an AbsoluteSystemPath -func UnsafeToAnchoredSystemPath(s string) turbopath.AnchoredSystemPath { - return turbopath.AnchoredSystemPath(s) -} - -// AbsoluteSystemPathFromUpstream is used to mark return values from APIs that we -// expect to give us absolute paths. No checking is performed. -// Prefer to use this over a cast to maintain the search-ability of interfaces -// into and out of the turbopath.AbsoluteSystemPath type. -func AbsoluteSystemPathFromUpstream(s string) turbopath.AbsoluteSystemPath { - return turbopath.AbsoluteSystemPath(s) -} - -// GetCwd returns the calculated working directory after traversing symlinks. -func GetCwd(cwdRaw string) (turbopath.AbsoluteSystemPath, error) { - if cwdRaw == "" { - var err error - cwdRaw, err = os.Getwd() - if err != nil { - return "", err - } - } - // We evaluate symlinks here because the package managers - // we support do the same. - cwdRaw, err := filepath.EvalSymlinks(cwdRaw) - if err != nil { - return "", fmt.Errorf("evaluating symlinks in cwd: %w", err) - } - cwd, err := CheckedToAbsoluteSystemPath(cwdRaw) - if err != nil { - return "", fmt.Errorf("cwd is not an absolute path %v: %v", cwdRaw, err) - } - return cwd, nil -} - -// GetVolumeRoot returns the root directory given an absolute path. -func GetVolumeRoot(absolutePath string) string { - return filepath.VolumeName(absolutePath) + string(os.PathSeparator) -} - -// CreateDirFSAtRoot creates an `os.dirFS` instance at the root of the -// volume containing the specified path. -func CreateDirFSAtRoot(absolutePath string) iofs.FS { - return os.DirFS(GetVolumeRoot(absolutePath)) -} - -// GetDirFSRootPath returns the root path of a os.dirFS. -func GetDirFSRootPath(fsys iofs.FS) string { - // We can't typecheck fsys to enforce using an `os.dirFS` because the - // type isn't exported from `os`. So instead, reflection. 🤷♂️ - - fsysType := reflect.TypeOf(fsys).Name() - if fsysType != "dirFS" { - // This is not a user error, fail fast - panic("GetDirFSRootPath must receive an os.dirFS") - } - - // The underlying type is a string; this is the original path passed in. - return reflect.ValueOf(fsys).String() -} - -// IofsRelativePath calculates a `os.dirFS`-friendly path from an absolute system path. -func IofsRelativePath(fsysRoot string, absolutePath string) (string, error) { - return filepath.Rel(fsysRoot, absolutePath) -} - -// TempDir returns the absolute path of a directory with the given name -// under the system's default temp directory location -func TempDir(subDir string) turbopath.AbsoluteSystemPath { - return turbopath.AbsoluteSystemPath(os.TempDir()).UntypedJoin(subDir) -} - -// GetUserConfigDir returns the platform-specific common location -// for configuration files that belong to a user. -func GetUserConfigDir() turbopath.AbsoluteSystemPath { - configHome := AbsoluteSystemPathFromUpstream(xdg.ConfigHome) - return configHome.UntypedJoin("turborepo") -} diff --git a/cli/internal/fs/testdata/both/package.json b/cli/internal/fs/testdata/both/package.json deleted file mode 100644 index 03534b7..0000000 --- a/cli/internal/fs/testdata/both/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "turbo": { - "pipeline": { - "build": {} - } - } -} diff --git a/cli/internal/fs/testdata/both/turbo.json b/cli/internal/fs/testdata/both/turbo.json deleted file mode 100644 index 721e897..0000000 --- a/cli/internal/fs/testdata/both/turbo.json +++ /dev/null @@ -1,18 +0,0 @@ -// mocked test comment -{ - "pipeline": { - "build": { - // mocked test comment - "dependsOn": [ - // mocked test comment - "^build" - ], - "outputs": ["dist/**", ".next/**", "!dist/assets/**"], - "outputMode": "new-only" - } // mocked test comment - }, - "remoteCache": { - "teamId": "team_id", - "signature": true - } -} diff --git a/cli/internal/fs/testdata/correct/turbo.json b/cli/internal/fs/testdata/correct/turbo.json deleted file mode 100644 index e22cde2..0000000 --- a/cli/internal/fs/testdata/correct/turbo.json +++ /dev/null @@ -1,49 +0,0 @@ -// mocked test comment -{ - "pipeline": { - "build": { - "experimentalPassthroughEnv": ["GITHUB_TOKEN"], - // mocked test comment - "dependsOn": [ - // mocked test comment - "^build" - ], - "outputs": ["dist/**", "!dist/assets/**", ".next/**"], - "outputMode": "new-only" - }, // mocked test comment - "lint": { - "outputs": [], - "dependsOn": ["$MY_VAR"], - "cache": true, - "outputMode": "new-only" - }, - "dev": { - "cache": false, - "outputMode": "full" - }, - /* mocked test comment */ - "publish": { - "outputs": ["dist/**"], - "inputs": [ - /* - mocked test comment - */ - "build/**/*" - ], - "dependsOn": [ - /* mocked test comment */ "^publish", - "^build", - "build", - "admin#lint" - ], - "cache": false - } - }, - "globalDependencies": ["some-file", "../another-dir/**", "$GLOBAL_ENV_VAR"], - "globlaEnv": ["SOME_VAR", "ANOTHER_VAR"], - "experimentalGlobalPassThroughEnv": ["AWS_SECRET_KEY"], - "remoteCache": { - "teamId": "team_id", - "signature": true - } -} diff --git a/cli/internal/fs/testdata/invalid-env-1/turbo.json b/cli/internal/fs/testdata/invalid-env-1/turbo.json deleted file mode 100644 index e4a6517..0000000 --- a/cli/internal/fs/testdata/invalid-env-1/turbo.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "pipeline": { - "task1": { - // all invalid value - "env": ["$A", "$B"] - } - } -} diff --git a/cli/internal/fs/testdata/invalid-env-2/turbo.json b/cli/internal/fs/testdata/invalid-env-2/turbo.json deleted file mode 100644 index 92eec96..0000000 --- a/cli/internal/fs/testdata/invalid-env-2/turbo.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "pipeline": { - "task1": { - // Mixed values - "env": ["$A", "B"] - } - } -} diff --git a/cli/internal/fs/testdata/invalid-global-env/turbo.json b/cli/internal/fs/testdata/invalid-global-env/turbo.json deleted file mode 100644 index 2ae9ff9..0000000 --- a/cli/internal/fs/testdata/invalid-global-env/turbo.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - // Both global declarations with duplicates - "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"], - // some invalid values - "globalEnv": ["FOO", "BAZ", "$QUX"], - "pipeline": { - "task1": { - "dependsOn": ["$A"] - } - } -} diff --git a/cli/internal/fs/testdata/legacy-env/turbo.json b/cli/internal/fs/testdata/legacy-env/turbo.json deleted file mode 100644 index 6b082c4..0000000 --- a/cli/internal/fs/testdata/legacy-env/turbo.json +++ /dev/null @@ -1,34 +0,0 @@ -// mocked test comment -{ - // Both global declarations with duplicates and with - "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"], - "globalEnv": ["FOO", "BAZ", "QUX"], - "pipeline": { - // Only legacy declaration - "task1": { - "dependsOn": ["$A"] - }, - // Only new declaration - "task2": { - "env": ["A"] - }, - // Same var declared in both - "task3": { - "dependsOn": ["$A"], - "env": ["A"] - }, - // Different vars declared in both - "task4": { - "dependsOn": ["$A"], - "env": ["B"] - }, - - // some edge cases - "task6": { "env": ["A", "B", "C"], "dependsOn": ["$D", "$E", "$F"] }, - "task7": { "env": ["A", "B", "C"], "dependsOn": ["$A", "$B", "$C"] }, - "task8": { "env": ["A", "B", "C"], "dependsOn": ["A", "B", "C"] }, - "task9": { "env": [], "dependsOn": ["$A"] }, - "task10": { "env": ["A", "A"], "dependsOn": ["$A", "$A"] }, - "task11": { "env": ["A", "A"], "dependsOn": ["$B", "$B"] } - } -} diff --git a/cli/internal/fs/testdata/legacy-only/package.json b/cli/internal/fs/testdata/legacy-only/package.json deleted file mode 100644 index 03534b7..0000000 --- a/cli/internal/fs/testdata/legacy-only/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "turbo": { - "pipeline": { - "build": {} - } - } -} diff --git a/cli/internal/fs/turbo_json.go b/cli/internal/fs/turbo_json.go deleted file mode 100644 index 71ef29d..0000000 --- a/cli/internal/fs/turbo_json.go +++ /dev/null @@ -1,741 +0,0 @@ -package fs - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "os" - "path/filepath" - "sort" - "strings" - - "github.com/muhammadmuzzammil1998/jsonc" - "github.com/pkg/errors" - "github.com/vercel/turbo/cli/internal/turbopath" - "github.com/vercel/turbo/cli/internal/util" -) - -const ( - configFile = "turbo.json" - envPipelineDelimiter = "$" - topologicalPipelineDelimiter = "^" -) - -type rawTurboJSON struct { - // Global root filesystem dependencies - GlobalDependencies []string `json:"globalDependencies,omitempty"` - // Global env - GlobalEnv []string `json:"globalEnv,omitempty"` - - // Global passthrough env - GlobalPassthroughEnv []string `json:"experimentalGlobalPassThroughEnv,omitempty"` - - // Pipeline is a map of Turbo pipeline entries which define the task graph - // and cache behavior on a per task or per package-task basis. - Pipeline Pipeline `json:"pipeline"` - // Configuration options when interfacing with the remote cache - RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"` - - // Extends can be the name of another workspace - Extends []string `json:"extends,omitempty"` -} - -// pristineTurboJSON is used when marshaling a TurboJSON object into a turbo.json string -// Notably, it includes a PristinePipeline instead of the regular Pipeline. (i.e. TaskDefinition -// instead of BookkeepingTaskDefinition.) -type pristineTurboJSON struct { - GlobalDependencies []string `json:"globalDependencies,omitempty"` - GlobalEnv []string `json:"globalEnv,omitempty"` - GlobalPassthroughEnv []string `json:"experimentalGlobalPassThroughEnv,omitempty"` - Pipeline PristinePipeline `json:"pipeline"` - RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"` - Extends []string `json:"extends,omitempty"` -} - -// TurboJSON represents a turbo.json configuration file -type TurboJSON struct { - GlobalDeps []string - GlobalEnv []string - GlobalPassthroughEnv []string - Pipeline Pipeline - RemoteCacheOptions RemoteCacheOptions - - // A list of Workspace names - Extends []string -} - -// RemoteCacheOptions is a struct for deserializing .remoteCache of configFile -type RemoteCacheOptions struct { - TeamID string `json:"teamId,omitempty"` - Signature bool `json:"signature,omitempty"` -} - -// rawTaskWithDefaults exists to Marshal (i.e. turn a TaskDefinition into json). -// We use this for printing ResolvedTaskConfiguration, because we _want_ to show -// the user the default values for key they have not configured. -type rawTaskWithDefaults struct { - Outputs []string `json:"outputs"` - Cache *bool `json:"cache"` - DependsOn []string `json:"dependsOn"` - Inputs []string `json:"inputs"` - OutputMode util.TaskOutputMode `json:"outputMode"` - PassthroughEnv []string `json:"experimentalPassThroughEnv,omitempty"` - Env []string `json:"env"` - Persistent bool `json:"persistent"` -} - -// rawTask exists to Unmarshal from json. When fields are omitted, we _want_ -// them to be missing, so that we can distinguish missing from empty value. -type rawTask struct { - Outputs []string `json:"outputs,omitempty"` - Cache *bool `json:"cache,omitempty"` - DependsOn []string `json:"dependsOn,omitempty"` - Inputs []string `json:"inputs,omitempty"` - OutputMode *util.TaskOutputMode `json:"outputMode,omitempty"` - Env []string `json:"env,omitempty"` - PassthroughEnv []string `json:"experimentalPassthroughEnv,omitempty"` - Persistent *bool `json:"persistent,omitempty"` -} - -// taskDefinitionHashable exists as a definition for PristinePipeline, which is used down -// stream for calculating the global hash. We want to exclude experimental fields here -// because we don't want experimental fields to be part of the global hash. -type taskDefinitionHashable struct { - Outputs TaskOutputs - ShouldCache bool - EnvVarDependencies []string - TopologicalDependencies []string - TaskDependencies []string - Inputs []string - OutputMode util.TaskOutputMode - Persistent bool -} - -// taskDefinitionExperiments is a list of config fields in a task definition that are considered -// experimental. We keep these separated so we can compute a global hash without these. -type taskDefinitionExperiments struct { - PassthroughEnv []string -} - -// PristinePipeline is a map of task names to TaskDefinition or taskDefinitionHashable. -// Depending on whether any experimental fields are defined, we will use either struct. -// The purpose is to omit experimental fields when making a pristine version, so that -// it doesn't show up in --dry/--summarize output or affect the global hash. -type PristinePipeline map[string]interface{} - -// Pipeline is a struct for deserializing .pipeline in configFile -type Pipeline map[string]BookkeepingTaskDefinition - -// BookkeepingTaskDefinition holds the underlying TaskDefinition and some bookkeeping data -// about the TaskDefinition. This wrapper struct allows us to leave TaskDefinition untouched. -type BookkeepingTaskDefinition struct { - definedFields util.Set - experimentalFields util.Set - experimental taskDefinitionExperiments - TaskDefinition taskDefinitionHashable -} - -// TaskDefinition is a representation of the configFile pipeline for further computation. -type TaskDefinition struct { - Outputs TaskOutputs - ShouldCache bool - - // This field is custom-marshalled from rawTask.Env and rawTask.DependsOn - EnvVarDependencies []string - - // rawTask.PassthroughEnv - PassthroughEnv []string - - // TopologicalDependencies are tasks from package dependencies. - // E.g. "build" is a topological dependency in: - // dependsOn: ['^build']. - // This field is custom-marshalled from rawTask.DependsOn - TopologicalDependencies []string - - // TaskDependencies are anything that is not a topological dependency - // E.g. both something and //whatever are TaskDependencies in: - // dependsOn: ['something', '//whatever'] - // This field is custom-marshalled from rawTask.DependsOn - TaskDependencies []string - - // Inputs indicate the list of files this Task depends on. If any of those files change - // we can conclude that any cached outputs or logs for this Task should be invalidated. - Inputs []string - - // OutputMode determins how we should log the output. - OutputMode util.TaskOutputMode - - // Persistent indicates whether the Task is expected to exit or not - // Tasks marked Persistent do not exit (e.g. --watch mode or dev servers) - Persistent bool -} - -// GetTask returns a TaskDefinition based on the ID (package#task format) or name (e.g. "build") -func (pc Pipeline) GetTask(taskID string, taskName string) (*BookkeepingTaskDefinition, error) { - // first check for package-tasks - taskDefinition, ok := pc[taskID] - if !ok { - // then check for regular tasks - fallbackTaskDefinition, notcool := pc[taskName] - // if neither, then bail - if !notcool { - // Return an empty TaskDefinition - return nil, fmt.Errorf("Could not find task \"%s\" in pipeline", taskID) - } - - // override if we need to... - taskDefinition = fallbackTaskDefinition - } - - return &taskDefinition, nil -} - -// LoadTurboConfig loads, or optionally, synthesizes a TurboJSON instance -func LoadTurboConfig(dir turbopath.AbsoluteSystemPath, rootPackageJSON *PackageJSON, includeSynthesizedFromRootPackageJSON bool) (*TurboJSON, error) { - // If the root package.json stil has a `turbo` key, log a warning and remove it. - if rootPackageJSON.LegacyTurboConfig != nil { - log.Printf("[WARNING] \"turbo\" in package.json is no longer supported. Migrate to %s by running \"npx @turbo/codemod create-turbo-config\"\n", configFile) - rootPackageJSON.LegacyTurboConfig = nil - } - - var turboJSON *TurboJSON - turboFromFiles, err := readTurboConfig(dir.UntypedJoin(configFile)) - - if !includeSynthesizedFromRootPackageJSON && err != nil { - // If the file didn't exist, throw a custom error here instead of propagating - if errors.Is(err, os.ErrNotExist) { - return nil, errors.Wrap(err, fmt.Sprintf("Could not find %s. Follow directions at https://turbo.build/repo/docs to create one", configFile)) - - } - - // There was an error, and we don't have any chance of recovering - // because we aren't synthesizing anything - return nil, err - } else if !includeSynthesizedFromRootPackageJSON { - // We're not synthesizing anything and there was no error, we're done - return turboFromFiles, nil - } else if errors.Is(err, os.ErrNotExist) { - // turbo.json doesn't exist, but we're going try to synthesize something - turboJSON = &TurboJSON{ - Pipeline: make(Pipeline), - } - } else if err != nil { - // some other happened, we can't recover - return nil, err - } else { - // we're synthesizing, but we have a starting point - // Note: this will have to change to support task inference in a monorepo - // for now, we're going to error on any "root" tasks and turn non-root tasks into root tasks - pipeline := make(Pipeline) - for taskID, taskDefinition := range turboFromFiles.Pipeline { - if util.IsPackageTask(taskID) { - return nil, fmt.Errorf("Package tasks (<package>#<task>) are not allowed in single-package repositories: found %v", taskID) - } - pipeline[util.RootTaskID(taskID)] = taskDefinition - } - turboJSON = turboFromFiles - turboJSON.Pipeline = pipeline - } - - for scriptName := range rootPackageJSON.Scripts { - if !turboJSON.Pipeline.HasTask(scriptName) { - taskName := util.RootTaskID(scriptName) - // Explicitly set ShouldCache to false in this definition and add the bookkeeping fields - // so downstream we can pretend that it was set on purpose (as if read from a config file) - // rather than defaulting to the 0-value of a boolean field. - turboJSON.Pipeline[taskName] = BookkeepingTaskDefinition{ - definedFields: util.SetFromStrings([]string{"ShouldCache"}), - TaskDefinition: taskDefinitionHashable{ - ShouldCache: false, - }, - } - } - } - return turboJSON, nil -} - -// TurboJSONValidation is the signature for a validation function passed to Validate() -type TurboJSONValidation func(*TurboJSON) []error - -// Validate calls an array of validation functions on the TurboJSON struct. -// The validations can be customized by the caller. -func (tj *TurboJSON) Validate(validations []TurboJSONValidation) []error { - allErrors := []error{} - for _, validation := range validations { - errors := validation(tj) - allErrors = append(allErrors, errors...) - } - - return allErrors -} - -// TaskOutputs represents the patterns for including and excluding files from outputs -type TaskOutputs struct { - Inclusions []string - Exclusions []string -} - -// Sort contents of task outputs -func (to TaskOutputs) Sort() TaskOutputs { - var inclusions []string - var exclusions []string - copy(inclusions, to.Inclusions) - copy(exclusions, to.Exclusions) - sort.Strings(inclusions) - sort.Strings(exclusions) - return TaskOutputs{Inclusions: inclusions, Exclusions: exclusions} -} - -// readTurboConfig reads turbo.json from a provided path -func readTurboConfig(turboJSONPath turbopath.AbsoluteSystemPath) (*TurboJSON, error) { - // If the configFile exists, use that - if turboJSONPath.FileExists() { - turboJSON, err := readTurboJSON(turboJSONPath) - if err != nil { - return nil, fmt.Errorf("%s: %w", configFile, err) - } - - return turboJSON, nil - } - - // If there's no turbo.json, return an error. - return nil, os.ErrNotExist -} - -// readTurboJSON reads the configFile in to a struct -func readTurboJSON(path turbopath.AbsoluteSystemPath) (*TurboJSON, error) { - file, err := path.Open() - if err != nil { - return nil, err - } - var turboJSON *TurboJSON - data, err := ioutil.ReadAll(file) - if err != nil { - return nil, err - } - - err = jsonc.Unmarshal(data, &turboJSON) - - if err != nil { - return nil, err - } - - return turboJSON, nil -} - -// GetTaskDefinition returns a TaskDefinition from a serialized definition in configFile -func (pc Pipeline) GetTaskDefinition(taskID string) (TaskDefinition, bool) { - if entry, ok := pc[taskID]; ok { - return entry.GetTaskDefinition(), true - } - _, task := util.GetPackageTaskFromId(taskID) - entry, ok := pc[task] - return entry.GetTaskDefinition(), ok -} - -// HasTask returns true if the given task is defined in the pipeline, either directly or -// via a package task (`pkg#task`) -func (pc Pipeline) HasTask(task string) bool { - for key := range pc { - if key == task { - return true - } - if util.IsPackageTask(key) { - _, taskName := util.GetPackageTaskFromId(key) - if taskName == task { - return true - } - } - } - return false -} - -// Pristine returns a PristinePipeline, this is used for printing to console and pruning -func (pc Pipeline) Pristine() PristinePipeline { - pristine := PristinePipeline{} - for taskName, taskDef := range pc { - // If there are any experimental fields, we will include them with 0-values - // if there aren't, we will omit them entirely - if taskDef.hasExperimentalFields() { - pristine[taskName] = taskDef.GetTaskDefinition() // merges experimental fields in - } else { - pristine[taskName] = taskDef.TaskDefinition // has no experimental fields - } - } - return pristine -} - -// hasField checks the internal bookkeeping definedFields field to -// see whether a field was actually in the underlying turbo.json -// or whether it was initialized with its 0-value. -func (btd BookkeepingTaskDefinition) hasField(fieldName string) bool { - return btd.definedFields.Includes(fieldName) || btd.experimentalFields.Includes(fieldName) -} - -// hasExperimentalFields keeps track of whether any experimental fields were found -func (btd BookkeepingTaskDefinition) hasExperimentalFields() bool { - return len(btd.experimentalFields) > 0 -} - -// GetTaskDefinition gets a TaskDefinition by merging the experimental and non-experimental fields -// into a single representation to use downstream. -func (btd BookkeepingTaskDefinition) GetTaskDefinition() TaskDefinition { - return TaskDefinition{ - Outputs: btd.TaskDefinition.Outputs, - ShouldCache: btd.TaskDefinition.ShouldCache, - EnvVarDependencies: btd.TaskDefinition.EnvVarDependencies, - TopologicalDependencies: btd.TaskDefinition.TopologicalDependencies, - TaskDependencies: btd.TaskDefinition.TaskDependencies, - Inputs: btd.TaskDefinition.Inputs, - OutputMode: btd.TaskDefinition.OutputMode, - Persistent: btd.TaskDefinition.Persistent, - // From experimental fields - PassthroughEnv: btd.experimental.PassthroughEnv, - } -} - -// MergeTaskDefinitions accepts an array of BookkeepingTaskDefinitions and merges them into -// a single TaskDefinition. It uses the bookkeeping definedFields to determine which fields should -// be overwritten and when 0-values should be respected. -func MergeTaskDefinitions(taskDefinitions []BookkeepingTaskDefinition) (*TaskDefinition, error) { - // Start with an empty definition - mergedTaskDefinition := &TaskDefinition{} - - // Set the default, because the 0-value will be false, and if no turbo.jsons had - // this field set for this task, we want it to be true. - mergedTaskDefinition.ShouldCache = true - - // For each of the TaskDefinitions we know of, merge them in - for _, bookkeepingTaskDef := range taskDefinitions { - taskDef := bookkeepingTaskDef.GetTaskDefinition() - - if bookkeepingTaskDef.hasField("Outputs") { - mergedTaskDefinition.Outputs = taskDef.Outputs - } - - if bookkeepingTaskDef.hasField("ShouldCache") { - mergedTaskDefinition.ShouldCache = taskDef.ShouldCache - } - - if bookkeepingTaskDef.hasField("EnvVarDependencies") { - mergedTaskDefinition.EnvVarDependencies = taskDef.EnvVarDependencies - } - - if bookkeepingTaskDef.hasField("PassthroughEnv") { - mergedTaskDefinition.PassthroughEnv = taskDef.PassthroughEnv - } - - if bookkeepingTaskDef.hasField("DependsOn") { - mergedTaskDefinition.TopologicalDependencies = taskDef.TopologicalDependencies - } - - if bookkeepingTaskDef.hasField("DependsOn") { - mergedTaskDefinition.TaskDependencies = taskDef.TaskDependencies - } - - if bookkeepingTaskDef.hasField("Inputs") { - mergedTaskDefinition.Inputs = taskDef.Inputs - } - - if bookkeepingTaskDef.hasField("OutputMode") { - mergedTaskDefinition.OutputMode = taskDef.OutputMode - } - if bookkeepingTaskDef.hasField("Persistent") { - mergedTaskDefinition.Persistent = taskDef.Persistent - } - } - - return mergedTaskDefinition, nil -} - -// UnmarshalJSON deserializes a single task definition from -// turbo.json into a TaskDefinition struct -func (btd *BookkeepingTaskDefinition) UnmarshalJSON(data []byte) error { - task := rawTask{} - if err := json.Unmarshal(data, &task); err != nil { - return err - } - - btd.definedFields = util.Set{} - btd.experimentalFields = util.Set{} - - if task.Outputs != nil { - var inclusions []string - var exclusions []string - // Assign a bookkeeping field so we know that there really were - // outputs configured in the underlying config file. - btd.definedFields.Add("Outputs") - - for _, glob := range task.Outputs { - if strings.HasPrefix(glob, "!") { - if filepath.IsAbs(glob[1:]) { - log.Printf("[WARNING] Using an absolute path in \"outputs\" (%v) will not work and will be an error in a future version", glob) - } - exclusions = append(exclusions, glob[1:]) - } else { - if filepath.IsAbs(glob) { - log.Printf("[WARNING] Using an absolute path in \"outputs\" (%v) will not work and will be an error in a future version", glob) - } - inclusions = append(inclusions, glob) - } - } - - btd.TaskDefinition.Outputs = TaskOutputs{ - Inclusions: inclusions, - Exclusions: exclusions, - } - - sort.Strings(btd.TaskDefinition.Outputs.Inclusions) - sort.Strings(btd.TaskDefinition.Outputs.Exclusions) - } - - if task.Cache == nil { - btd.TaskDefinition.ShouldCache = true - } else { - btd.definedFields.Add("ShouldCache") - btd.TaskDefinition.ShouldCache = *task.Cache - } - - envVarDependencies := make(util.Set) - envVarPassthroughs := make(util.Set) - - btd.TaskDefinition.TopologicalDependencies = []string{} // TODO @mehulkar: this should be a set - btd.TaskDefinition.TaskDependencies = []string{} // TODO @mehulkar: this should be a set - - // If there was a dependsOn field, add the bookkeeping - // we don't care what's in the field, just that it was there - // We'll use this marker to overwrite while merging TaskDefinitions. - if task.DependsOn != nil { - btd.definedFields.Add("DependsOn") - } - - for _, dependency := range task.DependsOn { - if strings.HasPrefix(dependency, envPipelineDelimiter) { - log.Printf("[DEPRECATED] Declaring an environment variable in \"dependsOn\" is deprecated, found %s. Use the \"env\" key or use `npx @turbo/codemod migrate-env-var-dependencies`.\n", dependency) - envVarDependencies.Add(strings.TrimPrefix(dependency, envPipelineDelimiter)) - } else if strings.HasPrefix(dependency, topologicalPipelineDelimiter) { - // Note: This will get assigned multiple times in the loop, but we only care that it's true - btd.TaskDefinition.TopologicalDependencies = append(btd.TaskDefinition.TopologicalDependencies, strings.TrimPrefix(dependency, topologicalPipelineDelimiter)) - } else { - btd.TaskDefinition.TaskDependencies = append(btd.TaskDefinition.TaskDependencies, dependency) - } - } - - sort.Strings(btd.TaskDefinition.TaskDependencies) - sort.Strings(btd.TaskDefinition.TopologicalDependencies) - - // Append env key into EnvVarDependencies - if task.Env != nil { - btd.definedFields.Add("EnvVarDependencies") - if err := gatherEnvVars(task.Env, "env", &envVarDependencies); err != nil { - return err - } - } - - btd.TaskDefinition.EnvVarDependencies = envVarDependencies.UnsafeListOfStrings() - - sort.Strings(btd.TaskDefinition.EnvVarDependencies) - - if task.PassthroughEnv != nil { - btd.experimentalFields.Add("PassthroughEnv") - if err := gatherEnvVars(task.PassthroughEnv, "passthrougEnv", &envVarPassthroughs); err != nil { - return err - } - } - - btd.experimental.PassthroughEnv = envVarPassthroughs.UnsafeListOfStrings() - sort.Strings(btd.experimental.PassthroughEnv) - - if task.Inputs != nil { - // Note that we don't require Inputs to be sorted, we're going to - // hash the resulting files and sort that instead - btd.definedFields.Add("Inputs") - // TODO: during rust port, this should be moved to a post-parse validation step - for _, input := range task.Inputs { - if filepath.IsAbs(input) { - log.Printf("[WARNING] Using an absolute path in \"inputs\" (%v) will not work and will be an error in a future version", input) - } - } - btd.TaskDefinition.Inputs = task.Inputs - } - - if task.OutputMode != nil { - btd.definedFields.Add("OutputMode") - btd.TaskDefinition.OutputMode = *task.OutputMode - } - - if task.Persistent != nil { - btd.definedFields.Add("Persistent") - btd.TaskDefinition.Persistent = *task.Persistent - } else { - btd.TaskDefinition.Persistent = false - } - return nil -} - -// MarshalJSON serializes taskDefinitionHashable struct into json -func (c taskDefinitionHashable) MarshalJSON() ([]byte, error) { - task := makeRawTask( - c.Persistent, - c.ShouldCache, - c.OutputMode, - c.Inputs, - c.Outputs, - c.EnvVarDependencies, - c.TaskDependencies, - c.TopologicalDependencies, - ) - return json.Marshal(task) -} - -// MarshalJSON serializes TaskDefinition struct into json -func (c TaskDefinition) MarshalJSON() ([]byte, error) { - task := makeRawTask( - c.Persistent, - c.ShouldCache, - c.OutputMode, - c.Inputs, - c.Outputs, - c.EnvVarDependencies, - c.TaskDependencies, - c.TopologicalDependencies, - ) - - if len(c.PassthroughEnv) > 0 { - task.PassthroughEnv = append(task.PassthroughEnv, c.PassthroughEnv...) - } - sort.Strings(task.PassthroughEnv) - - return json.Marshal(task) -} - -// UnmarshalJSON deserializes the contents of turbo.json into a TurboJSON struct -func (c *TurboJSON) UnmarshalJSON(data []byte) error { - raw := &rawTurboJSON{} - if err := json.Unmarshal(data, &raw); err != nil { - return err - } - - envVarDependencies := make(util.Set) - envVarPassthroughs := make(util.Set) - globalFileDependencies := make(util.Set) - - if err := gatherEnvVars(raw.GlobalEnv, "globalEnv", &envVarDependencies); err != nil { - return err - } - if err := gatherEnvVars(raw.GlobalPassthroughEnv, "experimentalGlobalPassThroughEnv", &envVarPassthroughs); err != nil { - return err - } - - // TODO: In the rust port, warnings should be refactored to a post-parse validation step - for _, value := range raw.GlobalDependencies { - if strings.HasPrefix(value, envPipelineDelimiter) { - log.Printf("[DEPRECATED] Declaring an environment variable in \"globalDependencies\" is deprecated, found %s. Use the \"globalEnv\" key or use `npx @turbo/codemod migrate-env-var-dependencies`.\n", value) - envVarDependencies.Add(strings.TrimPrefix(value, envPipelineDelimiter)) - } else { - if filepath.IsAbs(value) { - log.Printf("[WARNING] Using an absolute path in \"globalDependencies\" (%v) will not work and will be an error in a future version", value) - } - globalFileDependencies.Add(value) - } - } - - // turn the set into an array and assign to the TurboJSON struct fields. - c.GlobalEnv = envVarDependencies.UnsafeListOfStrings() - sort.Strings(c.GlobalEnv) - - if raw.GlobalPassthroughEnv != nil { - c.GlobalPassthroughEnv = envVarPassthroughs.UnsafeListOfStrings() - sort.Strings(c.GlobalPassthroughEnv) - } - - c.GlobalDeps = globalFileDependencies.UnsafeListOfStrings() - sort.Strings(c.GlobalDeps) - - // copy these over, we don't need any changes here. - c.Pipeline = raw.Pipeline - c.RemoteCacheOptions = raw.RemoteCacheOptions - c.Extends = raw.Extends - - return nil -} - -// MarshalJSON converts a TurboJSON into the equivalent json object in bytes -// note: we go via rawTurboJSON so that the output format is correct. -// This is used by `turbo prune` to generate a pruned turbo.json -// and also by --summarize & --dry=json to serialize the known config -// into something we can print to screen -func (c *TurboJSON) MarshalJSON() ([]byte, error) { - raw := pristineTurboJSON{} - raw.GlobalDependencies = c.GlobalDeps - raw.GlobalEnv = c.GlobalEnv - raw.GlobalPassthroughEnv = c.GlobalPassthroughEnv - raw.Pipeline = c.Pipeline.Pristine() - raw.RemoteCacheOptions = c.RemoteCacheOptions - - return json.Marshal(&raw) -} - -func makeRawTask(persistent bool, shouldCache bool, outputMode util.TaskOutputMode, inputs []string, outputs TaskOutputs, envVarDependencies []string, taskDependencies []string, topologicalDependencies []string) *rawTaskWithDefaults { - // Initialize with empty arrays, so we get empty arrays serialized into JSON - task := &rawTaskWithDefaults{ - Outputs: []string{}, - Inputs: []string{}, - Env: []string{}, - PassthroughEnv: []string{}, - DependsOn: []string{}, - } - - task.Persistent = persistent - task.Cache = &shouldCache - task.OutputMode = outputMode - - if len(inputs) > 0 { - task.Inputs = inputs - } - - if len(envVarDependencies) > 0 { - task.Env = append(task.Env, envVarDependencies...) - } - - if len(outputs.Inclusions) > 0 { - task.Outputs = append(task.Outputs, outputs.Inclusions...) - } - - for _, i := range outputs.Exclusions { - task.Outputs = append(task.Outputs, "!"+i) - } - - if len(taskDependencies) > 0 { - task.DependsOn = append(task.DependsOn, taskDependencies...) - } - - for _, i := range topologicalDependencies { - task.DependsOn = append(task.DependsOn, "^"+i) - } - - // These _should_ already be sorted when the TaskDefinition struct was unmarshaled, - // but we want to ensure they're sorted on the way out also, just in case something - // in the middle mutates the items. - sort.Strings(task.DependsOn) - sort.Strings(task.Outputs) - sort.Strings(task.Env) - sort.Strings(task.Inputs) - return task -} - -// gatherEnvVars puts env vars into the provided set as long as they don't have an invalid value. -func gatherEnvVars(vars []string, key string, into *util.Set) error { - for _, value := range vars { - if strings.HasPrefix(value, envPipelineDelimiter) { - // Hard error to help people specify this correctly during migration. - // TODO: Remove this error after we have run summary. - return fmt.Errorf("You specified \"%s\" in the \"%s\" key. You should not prefix your environment variables with \"%s\"", value, key, envPipelineDelimiter) - } - - into.Add(value) - } - - return nil -} diff --git a/cli/internal/fs/turbo_json_test.go b/cli/internal/fs/turbo_json_test.go deleted file mode 100644 index 1d384d5..0000000 --- a/cli/internal/fs/turbo_json_test.go +++ /dev/null @@ -1,277 +0,0 @@ -package fs - -import ( - "os" - "reflect" - "sort" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/vercel/turbo/cli/internal/turbopath" - "github.com/vercel/turbo/cli/internal/util" - "gotest.tools/v3/assert/cmp" -) - -func assertIsSorted(t *testing.T, arr []string, msg string) { - t.Helper() - if arr == nil { - return - } - - copied := make([]string, len(arr)) - copy(copied, arr) - sort.Strings(copied) - if !reflect.DeepEqual(arr, copied) { - t.Errorf("Expected sorted, got %v: %v", arr, msg) - } -} - -func Test_ReadTurboConfig(t *testing.T) { - testDir := getTestDir(t, "correct") - turboJSON, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) - - if turboJSONReadErr != nil { - t.Fatalf("invalid parse: %#v", turboJSONReadErr) - } - - assert.EqualValues(t, []string{"AWS_SECRET_KEY"}, turboJSON.GlobalPassthroughEnv) - - pipelineExpected := map[string]BookkeepingTaskDefinition{ - "build": { - definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "DependsOn"}), - experimentalFields: util.SetFromStrings([]string{"PassthroughEnv"}), - experimental: taskDefinitionExperiments{ - PassthroughEnv: []string{"GITHUB_TOKEN"}, - }, - TaskDefinition: taskDefinitionHashable{ - Outputs: TaskOutputs{Inclusions: []string{".next/**", "dist/**"}, Exclusions: []string{"dist/assets/**"}}, - TopologicalDependencies: []string{"build"}, - EnvVarDependencies: []string{}, - TaskDependencies: []string{}, - ShouldCache: true, - OutputMode: util.NewTaskOutput, - }, - }, - "lint": { - definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "ShouldCache", "DependsOn"}), - experimentalFields: util.SetFromStrings([]string{}), - experimental: taskDefinitionExperiments{ - PassthroughEnv: []string{}, - }, - TaskDefinition: taskDefinitionHashable{ - Outputs: TaskOutputs{}, - TopologicalDependencies: []string{}, - EnvVarDependencies: []string{"MY_VAR"}, - TaskDependencies: []string{}, - ShouldCache: true, - OutputMode: util.NewTaskOutput, - }, - }, - "dev": { - definedFields: util.SetFromStrings([]string{"OutputMode", "ShouldCache"}), - experimentalFields: util.SetFromStrings([]string{}), - experimental: taskDefinitionExperiments{ - PassthroughEnv: []string{}, - }, - TaskDefinition: taskDefinitionHashable{ - Outputs: TaskOutputs{}, - TopologicalDependencies: []string{}, - EnvVarDependencies: []string{}, - TaskDependencies: []string{}, - ShouldCache: false, - OutputMode: util.FullTaskOutput, - }, - }, - "publish": { - definedFields: util.SetFromStrings([]string{"Inputs", "Outputs", "DependsOn", "ShouldCache"}), - experimentalFields: util.SetFromStrings([]string{}), - experimental: taskDefinitionExperiments{ - PassthroughEnv: []string{}, - }, - TaskDefinition: taskDefinitionHashable{ - Outputs: TaskOutputs{Inclusions: []string{"dist/**"}}, - TopologicalDependencies: []string{"build", "publish"}, - EnvVarDependencies: []string{}, - TaskDependencies: []string{"admin#lint", "build"}, - ShouldCache: false, - Inputs: []string{"build/**/*"}, - OutputMode: util.FullTaskOutput, - }, - }, - } - - validateOutput(t, turboJSON, pipelineExpected) - remoteCacheOptionsExpected := RemoteCacheOptions{"team_id", true} - assert.EqualValues(t, remoteCacheOptionsExpected, turboJSON.RemoteCacheOptions) -} - -func Test_LoadTurboConfig_Legacy(t *testing.T) { - testDir := getTestDir(t, "legacy-only") - packageJSONPath := testDir.UntypedJoin("package.json") - rootPackageJSON, pkgJSONReadErr := ReadPackageJSON(packageJSONPath) - - if pkgJSONReadErr != nil { - t.Fatalf("invalid parse: %#v", pkgJSONReadErr) - } - - _, turboJSONReadErr := LoadTurboConfig(testDir, rootPackageJSON, false) - expectedErrorMsg := "Could not find turbo.json. Follow directions at https://turbo.build/repo/docs to create one: file does not exist" - assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) -} - -func Test_LoadTurboConfig_BothCorrectAndLegacy(t *testing.T) { - testDir := getTestDir(t, "both") - - packageJSONPath := testDir.UntypedJoin("package.json") - rootPackageJSON, pkgJSONReadErr := ReadPackageJSON(packageJSONPath) - - if pkgJSONReadErr != nil { - t.Fatalf("invalid parse: %#v", pkgJSONReadErr) - } - - turboJSON, turboJSONReadErr := LoadTurboConfig(testDir, rootPackageJSON, false) - - if turboJSONReadErr != nil { - t.Fatalf("invalid parse: %#v", turboJSONReadErr) - } - - pipelineExpected := map[string]BookkeepingTaskDefinition{ - "build": { - definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "DependsOn"}), - experimentalFields: util.SetFromStrings([]string{}), - experimental: taskDefinitionExperiments{ - PassthroughEnv: []string{}, - }, - TaskDefinition: taskDefinitionHashable{ - Outputs: TaskOutputs{Inclusions: []string{".next/**", "dist/**"}, Exclusions: []string{"dist/assets/**"}}, - TopologicalDependencies: []string{"build"}, - EnvVarDependencies: []string{}, - TaskDependencies: []string{}, - ShouldCache: true, - OutputMode: util.NewTaskOutput, - }, - }, - } - - validateOutput(t, turboJSON, pipelineExpected) - - remoteCacheOptionsExpected := RemoteCacheOptions{"team_id", true} - assert.EqualValues(t, remoteCacheOptionsExpected, turboJSON.RemoteCacheOptions) - assert.Equal(t, rootPackageJSON.LegacyTurboConfig == nil, true) -} - -func Test_ReadTurboConfig_InvalidEnvDeclarations1(t *testing.T) { - testDir := getTestDir(t, "invalid-env-1") - _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) - - expectedErrorMsg := "turbo.json: You specified \"$A\" in the \"env\" key. You should not prefix your environment variables with \"$\"" - assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) -} - -func Test_ReadTurboConfig_InvalidEnvDeclarations2(t *testing.T) { - testDir := getTestDir(t, "invalid-env-2") - _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) - expectedErrorMsg := "turbo.json: You specified \"$A\" in the \"env\" key. You should not prefix your environment variables with \"$\"" - assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) -} - -func Test_ReadTurboConfig_InvalidGlobalEnvDeclarations(t *testing.T) { - testDir := getTestDir(t, "invalid-global-env") - _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) - expectedErrorMsg := "turbo.json: You specified \"$QUX\" in the \"globalEnv\" key. You should not prefix your environment variables with \"$\"" - assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) -} - -func Test_ReadTurboConfig_EnvDeclarations(t *testing.T) { - testDir := getTestDir(t, "legacy-env") - turboJSON, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) - - if turboJSONReadErr != nil { - t.Fatalf("invalid parse: %#v", turboJSONReadErr) - } - - pipeline := turboJSON.Pipeline - assert.EqualValues(t, pipeline["task1"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) - assert.EqualValues(t, pipeline["task2"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) - assert.EqualValues(t, pipeline["task3"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) - assert.EqualValues(t, pipeline["task4"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B"})) - assert.EqualValues(t, pipeline["task6"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C", "D", "E", "F"})) - assert.EqualValues(t, pipeline["task7"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C"})) - assert.EqualValues(t, pipeline["task8"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C"})) - assert.EqualValues(t, pipeline["task9"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) - assert.EqualValues(t, pipeline["task10"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) - assert.EqualValues(t, pipeline["task11"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B"})) - - // check global env vars also - assert.EqualValues(t, sortedArray([]string{"FOO", "BAR", "BAZ", "QUX"}), sortedArray(turboJSON.GlobalEnv)) - assert.EqualValues(t, sortedArray([]string{"somefile.txt"}), sortedArray(turboJSON.GlobalDeps)) -} - -func Test_TaskOutputsSort(t *testing.T) { - inclusions := []string{"foo/**", "bar"} - exclusions := []string{"special-file", ".hidden/**"} - taskOutputs := TaskOutputs{Inclusions: inclusions, Exclusions: exclusions} - sortedOutputs := taskOutputs.Sort() - assertIsSorted(t, sortedOutputs.Inclusions, "Inclusions") - assertIsSorted(t, sortedOutputs.Exclusions, "Exclusions") - assert.False(t, cmp.DeepEqual(taskOutputs, sortedOutputs)().Success()) -} - -// Helpers -func validateOutput(t *testing.T, turboJSON *TurboJSON, expectedPipeline Pipeline) { - t.Helper() - assertIsSorted(t, turboJSON.GlobalDeps, "Global Deps") - assertIsSorted(t, turboJSON.GlobalEnv, "Global Env") - validatePipeline(t, turboJSON.Pipeline, expectedPipeline) -} - -func validatePipeline(t *testing.T, actual Pipeline, expected Pipeline) { - t.Helper() - // check top level keys - if len(actual) != len(expected) { - expectedKeys := []string{} - for k := range expected { - expectedKeys = append(expectedKeys, k) - } - actualKeys := []string{} - for k := range actual { - actualKeys = append(actualKeys, k) - } - t.Errorf("pipeline tasks mismatch. got %v, want %v", strings.Join(actualKeys, ","), strings.Join(expectedKeys, ",")) - } - - // check individual task definitions - for taskName, expectedTaskDefinition := range expected { - bookkeepingTaskDef, ok := actual[taskName] - if !ok { - t.Errorf("missing expected task: %v", taskName) - } - actualTaskDefinition := bookkeepingTaskDef.GetTaskDefinition() - assertIsSorted(t, actualTaskDefinition.Outputs.Inclusions, "Task output inclusions") - assertIsSorted(t, actualTaskDefinition.Outputs.Exclusions, "Task output exclusions") - assertIsSorted(t, actualTaskDefinition.EnvVarDependencies, "Task env vars") - assertIsSorted(t, actualTaskDefinition.PassthroughEnv, "Task env vars") - assertIsSorted(t, actualTaskDefinition.TopologicalDependencies, "Topo deps") - assertIsSorted(t, actualTaskDefinition.TaskDependencies, "Task deps") - assert.EqualValuesf(t, expectedTaskDefinition, bookkeepingTaskDef, "task definition mismatch for %v", taskName) - } -} - -func getTestDir(t *testing.T, testName string) turbopath.AbsoluteSystemPath { - defaultCwd, err := os.Getwd() - if err != nil { - t.Errorf("failed to get cwd: %v", err) - } - cwd, err := CheckedToAbsoluteSystemPath(defaultCwd) - if err != nil { - t.Fatalf("cwd is not an absolute directory %v: %v", defaultCwd, err) - } - - return cwd.UntypedJoin("testdata", testName) -} - -func sortedArray(arr []string) []string { - sort.Strings(arr) - return arr -} |
