From dd84b9d64fb98746a230cd24233ff50a562c39c9 Mon Sep 17 00:00:00 2001 From: 简律纯 Date: Fri, 28 Apr 2023 01:36:44 +0800 Subject: --- cli/internal/fs/copy_file.go | 81 +++ cli/internal/fs/copy_file_test.go | 198 ++++++ cli/internal/fs/fs.go | 191 ++++++ cli/internal/fs/fs_test.go | 60 ++ cli/internal/fs/fs_windows_test.go | 18 + cli/internal/fs/get_turbo_data_dir_go.go | 16 + cli/internal/fs/get_turbo_data_dir_rust.go | 16 + cli/internal/fs/hash.go | 61 ++ cli/internal/fs/hash_test.go | 53 ++ cli/internal/fs/lstat.go | 74 ++ cli/internal/fs/package_json.go | 142 ++++ cli/internal/fs/package_json_test.go | 174 +++++ cli/internal/fs/path.go | 113 ++++ cli/internal/fs/testdata/both/package.json | 7 + cli/internal/fs/testdata/both/turbo.json | 18 + cli/internal/fs/testdata/correct/turbo.json | 49 ++ cli/internal/fs/testdata/invalid-env-1/turbo.json | 8 + cli/internal/fs/testdata/invalid-env-2/turbo.json | 8 + .../fs/testdata/invalid-global-env/turbo.json | 11 + cli/internal/fs/testdata/legacy-env/turbo.json | 34 + cli/internal/fs/testdata/legacy-only/package.json | 7 + cli/internal/fs/turbo_json.go | 741 +++++++++++++++++++++ cli/internal/fs/turbo_json_test.go | 277 ++++++++ 23 files changed, 2357 insertions(+) create mode 100644 cli/internal/fs/copy_file.go create mode 100644 cli/internal/fs/copy_file_test.go create mode 100644 cli/internal/fs/fs.go create mode 100644 cli/internal/fs/fs_test.go create mode 100644 cli/internal/fs/fs_windows_test.go create mode 100644 cli/internal/fs/get_turbo_data_dir_go.go create mode 100644 cli/internal/fs/get_turbo_data_dir_rust.go create mode 100644 cli/internal/fs/hash.go create mode 100644 cli/internal/fs/hash_test.go create mode 100644 cli/internal/fs/lstat.go create mode 100644 cli/internal/fs/package_json.go create mode 100644 cli/internal/fs/package_json_test.go create mode 100644 cli/internal/fs/path.go create mode 100644 cli/internal/fs/testdata/both/package.json create mode 100644 cli/internal/fs/testdata/both/turbo.json create mode 100644 cli/internal/fs/testdata/correct/turbo.json create mode 100644 cli/internal/fs/testdata/invalid-env-1/turbo.json create mode 100644 cli/internal/fs/testdata/invalid-env-2/turbo.json create mode 100644 cli/internal/fs/testdata/invalid-global-env/turbo.json create mode 100644 cli/internal/fs/testdata/legacy-env/turbo.json create mode 100644 cli/internal/fs/testdata/legacy-only/package.json create mode 100644 cli/internal/fs/turbo_json.go create mode 100644 cli/internal/fs/turbo_json_test.go (limited to 'cli/internal/fs') diff --git a/cli/internal/fs/copy_file.go b/cli/internal/fs/copy_file.go new file mode 100644 index 0000000..e7619de --- /dev/null +++ b/cli/internal/fs/copy_file.go @@ -0,0 +1,81 @@ +// Adapted from https://github.com/thought-machine/please +// Copyright Thought Machine, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package fs + +import ( + "errors" + "os" + "path/filepath" + + "github.com/karrick/godirwalk" +) + +// RecursiveCopy copies either a single file or a directory. +// 'mode' is the mode of the destination file. +func RecursiveCopy(from string, to string) error { + // Verified all callers are passing in absolute paths for from (and to) + statedFrom := LstatCachedFile{Path: UnsafeToAbsoluteSystemPath(from)} + fromType, err := statedFrom.GetType() + if err != nil { + return err + } + + if fromType.IsDir() { + return WalkMode(statedFrom.Path.ToStringDuringMigration(), func(name string, isDir bool, fileType os.FileMode) error { + dest := filepath.Join(to, name[len(statedFrom.Path.ToString()):]) + // name is absolute, (originates from godirwalk) + src := LstatCachedFile{Path: UnsafeToAbsoluteSystemPath(name), fileType: &fileType} + if isDir { + mode, err := src.GetMode() + if err != nil { + return err + } + return os.MkdirAll(dest, mode) + } + return CopyFile(&src, dest) + }) + } + return CopyFile(&statedFrom, to) +} + +// Walk implements an equivalent to filepath.Walk. +// It's implemented over github.com/karrick/godirwalk but the provided interface doesn't use that +// to make it a little easier to handle. +func Walk(rootPath string, callback func(name string, isDir bool) error) error { + return WalkMode(rootPath, func(name string, isDir bool, mode os.FileMode) error { + return callback(name, isDir) + }) +} + +// WalkMode is like Walk but the callback receives an additional type specifying the file mode type. +// N.B. This only includes the bits of the mode that determine the mode type, not the permissions. +func WalkMode(rootPath string, callback func(name string, isDir bool, mode os.FileMode) error) error { + return godirwalk.Walk(rootPath, &godirwalk.Options{ + Callback: func(name string, info *godirwalk.Dirent) error { + // currently we support symlinked files, but not symlinked directories: + // For copying, we Mkdir and bail if we encounter a symlink to a directoy + // For finding packages, we enumerate the symlink, but don't follow inside + isDir, err := info.IsDirOrSymlinkToDir() + if err != nil { + pathErr := &os.PathError{} + if errors.As(err, &pathErr) { + // If we have a broken link, skip this entry + return godirwalk.SkipThis + } + return err + } + return callback(name, isDir, info.ModeType()) + }, + ErrorCallback: func(pathname string, err error) godirwalk.ErrorAction { + pathErr := &os.PathError{} + if errors.As(err, &pathErr) { + return godirwalk.SkipNode + } + return godirwalk.Halt + }, + Unsorted: true, + AllowNonDirectory: true, + FollowSymbolicLinks: false, + }) +} diff --git a/cli/internal/fs/copy_file_test.go b/cli/internal/fs/copy_file_test.go new file mode 100644 index 0000000..6a61576 --- /dev/null +++ b/cli/internal/fs/copy_file_test.go @@ -0,0 +1,198 @@ +package fs + +import ( + "errors" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/vercel/turbo/cli/internal/turbopath" + "gotest.tools/v3/assert" + "gotest.tools/v3/fs" +) + +func TestCopyFile(t *testing.T) { + srcTmpDir := turbopath.AbsoluteSystemPath(t.TempDir()) + destTmpDir := turbopath.AbsoluteSystemPath(t.TempDir()) + srcFilePath := srcTmpDir.UntypedJoin("src") + destFilePath := destTmpDir.UntypedJoin("dest") + from := &LstatCachedFile{Path: srcFilePath} + + // The src file doesn't exist, will error. + err := CopyFile(from, destFilePath.ToString()) + pathErr := &os.PathError{} + if !errors.As(err, &pathErr) { + t.Errorf("got %v, want PathError", err) + } + + // Create the src file. + srcFile, err := srcFilePath.Create() + assert.NilError(t, err, "Create") + _, err = srcFile.WriteString("src") + assert.NilError(t, err, "WriteString") + assert.NilError(t, srcFile.Close(), "Close") + + // Copy the src to the dest. + err = CopyFile(from, destFilePath.ToString()) + assert.NilError(t, err, "src exists dest does not, should not error.") + + // Now test for symlinks. + symlinkSrcDir := turbopath.AbsoluteSystemPath(t.TempDir()) + symlinkTargetDir := turbopath.AbsoluteSystemPath(t.TempDir()) + symlinkDestDir := turbopath.AbsoluteSystemPath(t.TempDir()) + symlinkSrcPath := symlinkSrcDir.UntypedJoin("symlink") + symlinkTargetPath := symlinkTargetDir.UntypedJoin("target") + symlinkDestPath := symlinkDestDir.UntypedJoin("dest") + fromSymlink := &LstatCachedFile{Path: symlinkSrcPath} + + // Create the symlink target. + symlinkTargetFile, err := symlinkTargetPath.Create() + assert.NilError(t, err, "Create") + _, err = symlinkTargetFile.WriteString("Target") + assert.NilError(t, err, "WriteString") + assert.NilError(t, symlinkTargetFile.Close(), "Close") + + // Link things up. + err = symlinkSrcPath.Symlink(symlinkTargetPath.ToString()) + assert.NilError(t, err, "Symlink") + + // Run the test. + err = CopyFile(fromSymlink, symlinkDestPath.ToString()) + assert.NilError(t, err, "Copying a valid symlink does not error.") + + // Break the symlink. + err = symlinkTargetPath.Remove() + assert.NilError(t, err, "breaking the symlink") + + // Remove the existing copy. + err = symlinkDestPath.Remove() + assert.NilError(t, err, "existing copy is removed") + + // Try copying the now-broken symlink. + err = CopyFile(fromSymlink, symlinkDestPath.ToString()) + assert.NilError(t, err, "CopyFile") + + // Confirm that it copied + target, err := symlinkDestPath.Readlink() + assert.NilError(t, err, "Readlink") + assert.Equal(t, target, symlinkTargetPath.ToString()) +} + +func TestCopyOrLinkFileWithPerms(t *testing.T) { + // Directory layout: + // + // / + // foo + readonlyMode := os.FileMode(0444) + srcDir := turbopath.AbsoluteSystemPath(t.TempDir()) + dstDir := turbopath.AbsoluteSystemPath(t.TempDir()) + srcFilePath := srcDir.UntypedJoin("src") + dstFilePath := dstDir.UntypedJoin("dst") + srcFile, err := srcFilePath.Create() + defer func() { _ = srcFile.Close() }() + assert.NilError(t, err, "Create") + err = srcFile.Chmod(readonlyMode) + assert.NilError(t, err, "Chmod") + err = CopyFile(&LstatCachedFile{Path: srcFilePath}, dstFilePath.ToStringDuringMigration()) + assert.NilError(t, err, "CopyOrLinkFile") + info, err := dstFilePath.Lstat() + assert.NilError(t, err, "Lstat") + assert.Equal(t, info.Mode(), readonlyMode, "expected dest to have matching permissions") +} + +func TestRecursiveCopy(t *testing.T) { + // Directory layout: + // + // / + // b + // child/ + // a + // link -> ../b + // broken -> missing + // circle -> ../child + src := fs.NewDir(t, "recursive-copy-or-link") + dst := fs.NewDir(t, "recursive-copy-or-link-dist") + childDir := filepath.Join(src.Path(), "child") + err := os.Mkdir(childDir, os.ModeDir|0777) + assert.NilError(t, err, "Mkdir") + aPath := filepath.Join(childDir, "a") + aFile, err := os.Create(aPath) + assert.NilError(t, err, "Create") + _, err = aFile.WriteString("hello") + assert.NilError(t, err, "WriteString") + assert.NilError(t, aFile.Close(), "Close") + + bPath := filepath.Join(src.Path(), "b") + bFile, err := os.Create(bPath) + assert.NilError(t, err, "Create") + _, err = bFile.WriteString("bFile") + assert.NilError(t, err, "WriteString") + assert.NilError(t, bFile.Close(), "Close") + + srcLinkPath := filepath.Join(childDir, "link") + assert.NilError(t, os.Symlink(filepath.FromSlash("../b"), srcLinkPath), "Symlink") + + srcBrokenLinkPath := filepath.Join(childDir, "broken") + assert.NilError(t, os.Symlink("missing", srcBrokenLinkPath), "Symlink") + circlePath := filepath.Join(childDir, "circle") + assert.NilError(t, os.Symlink(filepath.FromSlash("../child"), circlePath), "Symlink") + + err = RecursiveCopy(src.Path(), dst.Path()) + assert.NilError(t, err, "RecursiveCopy") + // For ensure multiple times copy will not broken + err = RecursiveCopy(src.Path(), dst.Path()) + assert.NilError(t, err, "RecursiveCopy") + + dstChildDir := filepath.Join(dst.Path(), "child") + assertDirMatches(t, childDir, dstChildDir) + dstAPath := filepath.Join(dst.Path(), "child", "a") + assertFileMatches(t, aPath, dstAPath) + dstBPath := filepath.Join(dst.Path(), "b") + assertFileMatches(t, bPath, dstBPath) + dstLinkPath := filepath.Join(dst.Path(), "child", "link") + dstLinkDest, err := os.Readlink(dstLinkPath) + assert.NilError(t, err, "Readlink") + expectedLinkDest := filepath.FromSlash("../b") + if dstLinkDest != expectedLinkDest { + t.Errorf("Readlink got %v, want %v", dstLinkDest, expectedLinkDest) + } + dstBrokenLinkPath := filepath.Join(dst.Path(), "child", "broken") + brokenLinkExists := PathExists(dstBrokenLinkPath) + if brokenLinkExists { + t.Errorf("We cached a broken link at %v", dstBrokenLinkPath) + } + // Currently, we convert symlink-to-directory to empty-directory + // This is very likely not ideal behavior, but leaving this test here to verify + // that it is what we expect at this point in time. + dstCirclePath := filepath.Join(dst.Path(), "child", "circle") + circleStat, err := os.Lstat(dstCirclePath) + assert.NilError(t, err, "Lstat") + assert.Equal(t, circleStat.IsDir(), true) + entries, err := os.ReadDir(dstCirclePath) + assert.NilError(t, err, "ReadDir") + assert.Equal(t, len(entries), 0) +} + +func assertFileMatches(t *testing.T, orig string, copy string) { + t.Helper() + origBytes, err := ioutil.ReadFile(orig) + assert.NilError(t, err, "ReadFile") + copyBytes, err := ioutil.ReadFile(copy) + assert.NilError(t, err, "ReadFile") + assert.DeepEqual(t, origBytes, copyBytes) + origStat, err := os.Lstat(orig) + assert.NilError(t, err, "Lstat") + copyStat, err := os.Lstat(copy) + assert.NilError(t, err, "Lstat") + assert.Equal(t, origStat.Mode(), copyStat.Mode()) +} + +func assertDirMatches(t *testing.T, orig string, copy string) { + t.Helper() + origStat, err := os.Lstat(orig) + assert.NilError(t, err, "Lstat") + copyStat, err := os.Lstat(copy) + assert.NilError(t, err, "Lstat") + assert.Equal(t, origStat.Mode(), copyStat.Mode()) +} diff --git a/cli/internal/fs/fs.go b/cli/internal/fs/fs.go new file mode 100644 index 0000000..77804c0 --- /dev/null +++ b/cli/internal/fs/fs.go @@ -0,0 +1,191 @@ +package fs + +import ( + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/pkg/errors" + "github.com/vercel/turbo/cli/internal/util" +) + +// https://github.com/thought-machine/please/blob/master/src/fs/fs.go + +// DirPermissions are the default permission bits we apply to directories. +const DirPermissions = os.ModeDir | 0775 + +// EnsureDir ensures that the directory of the given file has been created. +func EnsureDir(filename string) error { + dir := filepath.Dir(filename) + err := os.MkdirAll(dir, DirPermissions) + if err != nil && FileExists(dir) { + // It looks like this is a file and not a directory. Attempt to remove it; this can + // happen in some cases if you change a rule from outputting a file to a directory. + log.Printf("Attempting to remove file %s; a subdirectory is required", dir) + if err2 := os.Remove(dir); err2 == nil { + err = os.MkdirAll(dir, DirPermissions) + } else { + return err + } + } + return err +} + +var nonRelativeSentinel string = ".." + string(filepath.Separator) + +// DirContainsPath returns true if the path 'target' is contained within 'dir' +// Expects both paths to be absolute and does not verify that either path exists. +func DirContainsPath(dir string, target string) (bool, error) { + // On windows, trying to get a relative path between files on different volumes + // is an error. We don't care about the error, it's good enough for us to say + // that one path doesn't contain the other if they're on different volumes. + if runtime.GOOS == "windows" && filepath.VolumeName(dir) != filepath.VolumeName(target) { + return false, nil + } + // In Go, filepath.Rel can return a path that starts with "../" or equivalent. + // Checking filesystem-level contains can get extremely complicated + // (see https://github.com/golang/dep/blob/f13583b555deaa6742f141a9c1185af947720d60/internal/fs/fs.go#L33) + // As a compromise, rely on the stdlib to generate a relative path and then check + // if the first step is "../". + rel, err := filepath.Rel(dir, target) + if err != nil { + return false, err + } + return !strings.HasPrefix(rel, nonRelativeSentinel), nil +} + +// PathExists returns true if the given path exists, as a file or a directory. +func PathExists(filename string) bool { + _, err := os.Lstat(filename) + return err == nil +} + +// FileExists returns true if the given path exists and is a file. +func FileExists(filename string) bool { + info, err := os.Lstat(filename) + return err == nil && !info.IsDir() +} + +// CopyFile copies a file from 'from' to 'to', with an attempt to perform a copy & rename +// to avoid chaos if anything goes wrong partway. +func CopyFile(from *LstatCachedFile, to string) error { + fromMode, err := from.GetMode() + if err != nil { + return errors.Wrapf(err, "getting mode for %v", from.Path) + } + if fromMode&os.ModeSymlink != 0 { + target, err := from.Path.Readlink() + if err != nil { + return errors.Wrapf(err, "reading link target for %v", from.Path) + } + if err := EnsureDir(to); err != nil { + return err + } + if _, err := os.Lstat(to); err == nil { + // target link file exist, should remove it first + err := os.Remove(to) + if err != nil { + return err + } + } + return os.Symlink(target, to) + } + fromFile, err := from.Path.Open() + if err != nil { + return err + } + defer util.CloseAndIgnoreError(fromFile) + return writeFileFromStream(fromFile, to, fromMode) +} + +// writeFileFromStream writes data from a reader to the file named 'to', with an attempt to perform +// a copy & rename to avoid chaos if anything goes wrong partway. +func writeFileFromStream(fromFile io.Reader, to string, mode os.FileMode) error { + dir, file := filepath.Split(to) + if dir != "" { + if err := os.MkdirAll(dir, DirPermissions); err != nil { + return err + } + } + tempFile, err := ioutil.TempFile(dir, file) + if err != nil { + return err + } + if _, err := io.Copy(tempFile, fromFile); err != nil { + return err + } + if err := tempFile.Close(); err != nil { + return err + } + // OK, now file is written; adjust permissions appropriately. + if mode == 0 { + mode = 0664 + } + if err := os.Chmod(tempFile.Name(), mode); err != nil { + return err + } + // And move it to its final destination. + return renameFile(tempFile.Name(), to) +} + +// IsDirectory checks if a given path is a directory +func IsDirectory(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +// Try to gracefully rename the file as the os.Rename does not work across +// filesystems and on most Linux systems /tmp is mounted as tmpfs +func renameFile(from, to string) (err error) { + err = os.Rename(from, to) + if err == nil { + return nil + } + err = copyFile(from, to) + if err != nil { + return err + } + err = os.RemoveAll(from) + if err != nil { + return err + } + return nil +} + +func copyFile(from, to string) (err error) { + in, err := os.Open(from) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(to) + if err != nil { + return err + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + si, err := os.Stat(from) + if err != nil { + return err + } + err = os.Chmod(to, si.Mode()) + if err != nil { + return err + } + + return nil +} diff --git a/cli/internal/fs/fs_test.go b/cli/internal/fs/fs_test.go new file mode 100644 index 0000000..0598d43 --- /dev/null +++ b/cli/internal/fs/fs_test.go @@ -0,0 +1,60 @@ +package fs + +import ( + "path/filepath" + "testing" +) + +func Test_DirContainsPath(t *testing.T) { + parent, err := filepath.Abs(filepath.Join("some", "path")) + if err != nil { + t.Fatalf("failed to construct parent path %v", err) + } + testcases := []struct { + target []string + want bool + }{ + { + []string{"..", "elsewhere"}, + false, + }, + { + []string{"sibling"}, + false, + }, + { + // The same path as parent + []string{"some", "path"}, + true, + }, + { + []string{"some", "path", "..", "path", "inside", "parent"}, + true, + }, + { + []string{"some", "path", "inside", "..", "inside", "parent"}, + true, + }, + { + []string{"some", "path", "inside", "..", "..", "outside", "parent"}, + false, + }, + { + []string{"some", "pathprefix"}, + false, + }, + } + for _, tc := range testcases { + target, err := filepath.Abs(filepath.Join(tc.target...)) + if err != nil { + t.Fatalf("failed to construct path for %v: %v", tc.target, err) + } + got, err := DirContainsPath(parent, target) + if err != nil { + t.Fatalf("failed to check ") + } + if got != tc.want { + t.Errorf("DirContainsPath(%v, %v) got %v, want %v", parent, target, got, tc.want) + } + } +} diff --git a/cli/internal/fs/fs_windows_test.go b/cli/internal/fs/fs_windows_test.go new file mode 100644 index 0000000..4e71e2c --- /dev/null +++ b/cli/internal/fs/fs_windows_test.go @@ -0,0 +1,18 @@ +//go:build windows +// +build windows + +package fs + +import "testing" + +func TestDifferentVolumes(t *testing.T) { + p1 := "C:\\some\\path" + p2 := "D:\\other\\path" + contains, err := DirContainsPath(p1, p2) + if err != nil { + t.Errorf("DirContainsPath got error %v, want ", err) + } + if contains { + t.Errorf("DirContainsPath got true, want false") + } +} diff --git a/cli/internal/fs/get_turbo_data_dir_go.go b/cli/internal/fs/get_turbo_data_dir_go.go new file mode 100644 index 0000000..2cf459a --- /dev/null +++ b/cli/internal/fs/get_turbo_data_dir_go.go @@ -0,0 +1,16 @@ +//go:build go || !rust +// +build go !rust + +package fs + +import ( + "github.com/adrg/xdg" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// GetTurboDataDir returns a directory outside of the repo +// where turbo can store data files related to turbo. +func GetTurboDataDir() turbopath.AbsoluteSystemPath { + dataHome := AbsoluteSystemPathFromUpstream(xdg.DataHome) + return dataHome.UntypedJoin("turborepo") +} diff --git a/cli/internal/fs/get_turbo_data_dir_rust.go b/cli/internal/fs/get_turbo_data_dir_rust.go new file mode 100644 index 0000000..dbc80f3 --- /dev/null +++ b/cli/internal/fs/get_turbo_data_dir_rust.go @@ -0,0 +1,16 @@ +//go:build rust +// +build rust + +package fs + +import ( + "github.com/vercel/turbo/cli/internal/ffi" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// GetTurboDataDir returns a directory outside of the repo +// where turbo can store data files related to turbo. +func GetTurboDataDir() turbopath.AbsoluteSystemPath { + dir := ffi.GetTurboDataDir() + return turbopath.AbsoluteSystemPathFromUpstream(dir) +} diff --git a/cli/internal/fs/hash.go b/cli/internal/fs/hash.go new file mode 100644 index 0000000..fed7d87 --- /dev/null +++ b/cli/internal/fs/hash.go @@ -0,0 +1,61 @@ +package fs + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "os" + "strconv" + + "github.com/vercel/turbo/cli/internal/xxhash" +) + +func HashObject(i interface{}) (string, error) { + hash := xxhash.New() + + _, err := hash.Write([]byte(fmt.Sprintf("%v", i))) + + return hex.EncodeToString(hash.Sum(nil)), err +} + +func HashFile(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + hash := xxhash.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// GitLikeHashFile is a function that mimics how Git +// calculates the SHA1 for a file (or, in Git terms, a "blob") (without git) +func GitLikeHashFile(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + stat, err := file.Stat() + if err != nil { + return "", err + } + hash := sha1.New() + hash.Write([]byte("blob")) + hash.Write([]byte(" ")) + hash.Write([]byte(strconv.FormatInt(stat.Size(), 10))) + hash.Write([]byte{0}) + + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} diff --git a/cli/internal/fs/hash_test.go b/cli/internal/fs/hash_test.go new file mode 100644 index 0000000..dd2fa84 --- /dev/null +++ b/cli/internal/fs/hash_test.go @@ -0,0 +1,53 @@ +package fs + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +const _numOfRuns = 20 + +func Test_HashObjectStability(t *testing.T) { + type TestCase struct { + name string + obj interface{} + } + type complexStruct struct { + nested TaskOutputs + foo string + bar []string + } + + testCases := []TestCase{ + { + name: "task object", + obj: TaskOutputs{ + Inclusions: []string{"foo", "bar"}, + Exclusions: []string{"baz"}, + }, + }, + { + name: "complex struct", + obj: complexStruct{ + nested: TaskOutputs{ + Exclusions: []string{"bar", "baz"}, + Inclusions: []string{"foo"}, + }, + foo: "a", + bar: []string{"b", "c"}, + }, + }, + } + + for _, tc := range testCases { + expectedHash, err := HashObject(tc.obj) + assert.NilError(t, err, tc.name) + + for n := 0; n < _numOfRuns; n++ { + hash, err := HashObject(tc.obj) + assert.NilError(t, err, tc.name) + assert.Equal(t, expectedHash, hash, tc.name) + } + } +} diff --git a/cli/internal/fs/lstat.go b/cli/internal/fs/lstat.go new file mode 100644 index 0000000..eff0810 --- /dev/null +++ b/cli/internal/fs/lstat.go @@ -0,0 +1,74 @@ +package fs + +import ( + "io/fs" + "os" + + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// LstatCachedFile maintains a cache of file info, mode and type for the given Path +type LstatCachedFile struct { + Path turbopath.AbsoluteSystemPath + fileInfo fs.FileInfo + fileMode *fs.FileMode + fileType *fs.FileMode +} + +// GetInfo returns, and caches the file info for the LstatCachedFile.Path +func (file *LstatCachedFile) GetInfo() (fs.FileInfo, error) { + if file.fileInfo != nil { + return file.fileInfo, nil + } + + err := file.lstat() + if err != nil { + return nil, err + } + + return file.fileInfo, nil +} + +// GetMode returns, and caches the file mode for the LstatCachedFile.Path +func (file *LstatCachedFile) GetMode() (fs.FileMode, error) { + if file.fileMode != nil { + return *file.fileMode, nil + } + + err := file.lstat() + if err != nil { + return 0, err + } + + return *file.fileMode, nil +} + +// GetType returns, and caches the type bits of (FileMode & os.ModeType) for the LstatCachedFile.Path +func (file *LstatCachedFile) GetType() (fs.FileMode, error) { + if file.fileType != nil { + return *file.fileType, nil + } + + err := file.lstat() + if err != nil { + return 0, err + } + + return *file.fileType, nil +} + +func (file *LstatCachedFile) lstat() error { + fileInfo, err := file.Path.Lstat() + if err != nil { + return err + } + + fileMode := fileInfo.Mode() + fileModeType := fileMode & os.ModeType + + file.fileInfo = fileInfo + file.fileMode = &fileMode + file.fileType = &fileModeType + + return nil +} diff --git a/cli/internal/fs/package_json.go b/cli/internal/fs/package_json.go new file mode 100644 index 0000000..883f7a4 --- /dev/null +++ b/cli/internal/fs/package_json.go @@ -0,0 +1,142 @@ +package fs + +import ( + "bytes" + "encoding/json" + "sync" + + "github.com/vercel/turbo/cli/internal/lockfile" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// PackageJSON represents NodeJS package.json +type PackageJSON struct { + Name string `json:"name"` + Version string `json:"version"` + Scripts map[string]string `json:"scripts"` + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + OptionalDependencies map[string]string `json:"optionalDependencies"` + PeerDependencies map[string]string `json:"peerDependencies"` + PackageManager string `json:"packageManager"` + Os []string `json:"os"` + Workspaces Workspaces `json:"workspaces"` + Private bool `json:"private"` + // Exact JSON object stored in package.json including unknown fields + // During marshalling struct fields will take priority over raw fields + RawJSON map[string]interface{} `json:"-"` + + // relative path from repo root to the package.json file + PackageJSONPath turbopath.AnchoredSystemPath `json:"-"` + // relative path from repo root to the package + Dir turbopath.AnchoredSystemPath `json:"-"` + InternalDeps []string `json:"-"` + UnresolvedExternalDeps map[string]string `json:"-"` + TransitiveDeps []lockfile.Package `json:"-"` + LegacyTurboConfig *TurboJSON `json:"turbo"` + Mu sync.Mutex `json:"-"` + ExternalDepsHash string `json:"-"` +} + +type Workspaces []string + +type WorkspacesAlt struct { + Packages []string `json:"packages,omitempty"` +} + +func (r *Workspaces) UnmarshalJSON(data []byte) error { + var tmp = &WorkspacesAlt{} + if err := json.Unmarshal(data, tmp); err == nil { + *r = Workspaces(tmp.Packages) + return nil + } + var tempstr = []string{} + if err := json.Unmarshal(data, &tempstr); err != nil { + return err + } + *r = tempstr + return nil +} + +// ReadPackageJSON returns a struct of package.json +func ReadPackageJSON(path turbopath.AbsoluteSystemPath) (*PackageJSON, error) { + b, err := path.ReadFile() + if err != nil { + return nil, err + } + return UnmarshalPackageJSON(b) +} + +// UnmarshalPackageJSON decodes a byte slice into a PackageJSON struct +func UnmarshalPackageJSON(data []byte) (*PackageJSON, error) { + var rawJSON map[string]interface{} + if err := json.Unmarshal(data, &rawJSON); err != nil { + return nil, err + } + + pkgJSON := &PackageJSON{} + if err := json.Unmarshal(data, &pkgJSON); err != nil { + return nil, err + } + pkgJSON.RawJSON = rawJSON + + return pkgJSON, nil +} + +// MarshalPackageJSON Serialize PackageJSON to a slice of bytes +func MarshalPackageJSON(pkgJSON *PackageJSON) ([]byte, error) { + structuredContent, err := json.Marshal(pkgJSON) + if err != nil { + return nil, err + } + var structuredFields map[string]interface{} + if err := json.Unmarshal(structuredContent, &structuredFields); err != nil { + return nil, err + } + + fieldsToSerialize := make(map[string]interface{}, len(pkgJSON.RawJSON)) + + // copy pkgJSON.RawJSON + for key, value := range pkgJSON.RawJSON { + fieldsToSerialize[key] = value + } + + for key, value := range structuredFields { + if isEmpty(value) { + delete(fieldsToSerialize, key) + } else { + fieldsToSerialize[key] = value + } + } + + var b bytes.Buffer + encoder := json.NewEncoder(&b) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + if err := encoder.Encode(fieldsToSerialize); err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +func isEmpty(value interface{}) bool { + if value == nil { + return true + } + switch s := value.(type) { + case string: + return s == "" + case bool: + return !s + case []string: + return len(s) == 0 + case map[string]interface{}: + return len(s) == 0 + case Workspaces: + return len(s) == 0 + default: + // Assume any unknown types aren't empty + return false + } +} diff --git a/cli/internal/fs/package_json_test.go b/cli/internal/fs/package_json_test.go new file mode 100644 index 0000000..3c16620 --- /dev/null +++ b/cli/internal/fs/package_json_test.go @@ -0,0 +1,174 @@ +package fs + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func Test_UnmarshalPackageJSON(t *testing.T) { + type Case struct { + name string + json string + expectedFields *PackageJSON + } + + testCases := []Case{ + { + name: "basic types are in raw and processed", + json: `{"name":"foo","version":"1.2.3"}`, + expectedFields: &PackageJSON{ + Name: "foo", + Version: "1.2.3", + RawJSON: map[string]interface{}{ + "name": "foo", + "version": "1.2.3", + }, + }, + }, + { + name: "map types get copied", + json: `{"dependencies":{"foo":"1.2.3"},"devDependencies":{"bar": "^1.0.0"}}`, + expectedFields: &PackageJSON{ + Dependencies: map[string]string{"foo": "1.2.3"}, + DevDependencies: map[string]string{"bar": "^1.0.0"}, + RawJSON: map[string]interface{}{ + "dependencies": map[string]interface{}{"foo": "1.2.3"}, + "devDependencies": map[string]interface{}{"bar": "^1.0.0"}, + }, + }, + }, + { + name: "array types get copied", + json: `{"os":["linux", "windows"]}`, + expectedFields: &PackageJSON{ + Os: []string{"linux", "windows"}, + RawJSON: map[string]interface{}{ + "os": []interface{}{"linux", "windows"}, + }, + }, + }, + } + + for _, testCase := range testCases { + actual, err := UnmarshalPackageJSON([]byte(testCase.json)) + assert.NilError(t, err, testCase.name) + assertPackageJSONEqual(t, actual, testCase.expectedFields) + } +} + +func Test_MarshalPackageJSON(t *testing.T) { + type TestCase struct { + name string + input *PackageJSON + expected *PackageJSON + } + + testCases := []TestCase{ + { + name: "roundtrip should have no effect", + input: &PackageJSON{ + Name: "foo", + Version: "1.2.3", + RawJSON: map[string]interface{}{ + "name": "foo", + "version": "1.2.3", + }, + }, + expected: &PackageJSON{ + Name: "foo", + Version: "1.2.3", + RawJSON: map[string]interface{}{ + "name": "foo", + "version": "1.2.3", + }, + }, + }, + { + name: "structured fields should take priority over raw values", + input: &PackageJSON{ + Name: "foo", + Version: "2.3.4", + RawJSON: map[string]interface{}{ + "name": "foo", + "version": "1.2.3", + }, + }, + expected: &PackageJSON{ + Name: "foo", + Version: "2.3.4", + RawJSON: map[string]interface{}{ + "name": "foo", + "version": "2.3.4", + }, + }, + }, + { + name: "empty structured fields don't get serialized", + input: &PackageJSON{ + Name: "foo", + Version: "", + RawJSON: map[string]interface{}{ + "name": "foo", + "version": "1.2.3", + }, + }, + expected: &PackageJSON{ + Name: "foo", + Version: "", + RawJSON: map[string]interface{}{ + "name": "foo", + }, + }, + }, + { + name: "unstructured fields survive the round trip", + input: &PackageJSON{ + Name: "foo", + RawJSON: map[string]interface{}{ + "name": "foo", + "special-field": "special-value", + "special-config": map[string]interface{}{ + "flag": true, + "value": "toggled", + }, + }, + }, + expected: &PackageJSON{ + Name: "foo", + RawJSON: map[string]interface{}{ + "name": "foo", + "special-field": "special-value", + "special-config": map[string]interface{}{ + "flag": true, + "value": "toggled", + }, + }, + }, + }, + } + + for _, testCase := range testCases { + serializedInput, err := MarshalPackageJSON(testCase.input) + assert.NilError(t, err, testCase.name) + actual, err := UnmarshalPackageJSON(serializedInput) + assert.NilError(t, err, testCase.name) + assertPackageJSONEqual(t, actual, testCase.expected) + } +} + +// Asserts that the data section of two PackageJSON structs are equal +func assertPackageJSONEqual(t *testing.T, x *PackageJSON, y *PackageJSON) { + t.Helper() + assert.Equal(t, x.Name, y.Name) + assert.Equal(t, x.Version, y.Version) + assert.DeepEqual(t, x.Scripts, y.Scripts) + assert.DeepEqual(t, x.Dependencies, y.Dependencies) + assert.DeepEqual(t, x.DevDependencies, y.DevDependencies) + assert.DeepEqual(t, x.OptionalDependencies, y.OptionalDependencies) + assert.DeepEqual(t, x.PeerDependencies, y.PeerDependencies) + assert.Equal(t, x.PackageManager, y.PackageManager) + assert.DeepEqual(t, x.Workspaces, y.Workspaces) + assert.DeepEqual(t, x.Private, y.Private) + assert.DeepEqual(t, x.RawJSON, y.RawJSON) +} diff --git a/cli/internal/fs/path.go b/cli/internal/fs/path.go new file mode 100644 index 0000000..2023d69 --- /dev/null +++ b/cli/internal/fs/path.go @@ -0,0 +1,113 @@ +package fs + +import ( + "fmt" + iofs "io/fs" + "os" + "path/filepath" + "reflect" + + "github.com/adrg/xdg" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +// CheckedToAbsoluteSystemPath inspects a string and determines if it is an absolute path. +func CheckedToAbsoluteSystemPath(s string) (turbopath.AbsoluteSystemPath, error) { + if filepath.IsAbs(s) { + return turbopath.AbsoluteSystemPath(s), nil + } + return "", fmt.Errorf("%v is not an absolute path", s) +} + +// ResolveUnknownPath returns unknown if it is an absolute path, otherwise, it +// assumes unknown is a path relative to the given root. +func ResolveUnknownPath(root turbopath.AbsoluteSystemPath, unknown string) turbopath.AbsoluteSystemPath { + if filepath.IsAbs(unknown) { + return turbopath.AbsoluteSystemPath(unknown) + } + return root.UntypedJoin(unknown) +} + +// UnsafeToAbsoluteSystemPath directly converts a string to an AbsoluteSystemPath +func UnsafeToAbsoluteSystemPath(s string) turbopath.AbsoluteSystemPath { + return turbopath.AbsoluteSystemPath(s) +} + +// UnsafeToAnchoredSystemPath directly converts a string to an AbsoluteSystemPath +func UnsafeToAnchoredSystemPath(s string) turbopath.AnchoredSystemPath { + return turbopath.AnchoredSystemPath(s) +} + +// AbsoluteSystemPathFromUpstream is used to mark return values from APIs that we +// expect to give us absolute paths. No checking is performed. +// Prefer to use this over a cast to maintain the search-ability of interfaces +// into and out of the turbopath.AbsoluteSystemPath type. +func AbsoluteSystemPathFromUpstream(s string) turbopath.AbsoluteSystemPath { + return turbopath.AbsoluteSystemPath(s) +} + +// GetCwd returns the calculated working directory after traversing symlinks. +func GetCwd(cwdRaw string) (turbopath.AbsoluteSystemPath, error) { + if cwdRaw == "" { + var err error + cwdRaw, err = os.Getwd() + if err != nil { + return "", err + } + } + // We evaluate symlinks here because the package managers + // we support do the same. + cwdRaw, err := filepath.EvalSymlinks(cwdRaw) + if err != nil { + return "", fmt.Errorf("evaluating symlinks in cwd: %w", err) + } + cwd, err := CheckedToAbsoluteSystemPath(cwdRaw) + if err != nil { + return "", fmt.Errorf("cwd is not an absolute path %v: %v", cwdRaw, err) + } + return cwd, nil +} + +// GetVolumeRoot returns the root directory given an absolute path. +func GetVolumeRoot(absolutePath string) string { + return filepath.VolumeName(absolutePath) + string(os.PathSeparator) +} + +// CreateDirFSAtRoot creates an `os.dirFS` instance at the root of the +// volume containing the specified path. +func CreateDirFSAtRoot(absolutePath string) iofs.FS { + return os.DirFS(GetVolumeRoot(absolutePath)) +} + +// GetDirFSRootPath returns the root path of a os.dirFS. +func GetDirFSRootPath(fsys iofs.FS) string { + // We can't typecheck fsys to enforce using an `os.dirFS` because the + // type isn't exported from `os`. So instead, reflection. 🤷‍♂️ + + fsysType := reflect.TypeOf(fsys).Name() + if fsysType != "dirFS" { + // This is not a user error, fail fast + panic("GetDirFSRootPath must receive an os.dirFS") + } + + // The underlying type is a string; this is the original path passed in. + return reflect.ValueOf(fsys).String() +} + +// IofsRelativePath calculates a `os.dirFS`-friendly path from an absolute system path. +func IofsRelativePath(fsysRoot string, absolutePath string) (string, error) { + return filepath.Rel(fsysRoot, absolutePath) +} + +// TempDir returns the absolute path of a directory with the given name +// under the system's default temp directory location +func TempDir(subDir string) turbopath.AbsoluteSystemPath { + return turbopath.AbsoluteSystemPath(os.TempDir()).UntypedJoin(subDir) +} + +// GetUserConfigDir returns the platform-specific common location +// for configuration files that belong to a user. +func GetUserConfigDir() turbopath.AbsoluteSystemPath { + configHome := AbsoluteSystemPathFromUpstream(xdg.ConfigHome) + return configHome.UntypedJoin("turborepo") +} diff --git a/cli/internal/fs/testdata/both/package.json b/cli/internal/fs/testdata/both/package.json new file mode 100644 index 0000000..03534b7 --- /dev/null +++ b/cli/internal/fs/testdata/both/package.json @@ -0,0 +1,7 @@ +{ + "turbo": { + "pipeline": { + "build": {} + } + } +} diff --git a/cli/internal/fs/testdata/both/turbo.json b/cli/internal/fs/testdata/both/turbo.json new file mode 100644 index 0000000..721e897 --- /dev/null +++ b/cli/internal/fs/testdata/both/turbo.json @@ -0,0 +1,18 @@ +// mocked test comment +{ + "pipeline": { + "build": { + // mocked test comment + "dependsOn": [ + // mocked test comment + "^build" + ], + "outputs": ["dist/**", ".next/**", "!dist/assets/**"], + "outputMode": "new-only" + } // mocked test comment + }, + "remoteCache": { + "teamId": "team_id", + "signature": true + } +} diff --git a/cli/internal/fs/testdata/correct/turbo.json b/cli/internal/fs/testdata/correct/turbo.json new file mode 100644 index 0000000..e22cde2 --- /dev/null +++ b/cli/internal/fs/testdata/correct/turbo.json @@ -0,0 +1,49 @@ +// mocked test comment +{ + "pipeline": { + "build": { + "experimentalPassthroughEnv": ["GITHUB_TOKEN"], + // mocked test comment + "dependsOn": [ + // mocked test comment + "^build" + ], + "outputs": ["dist/**", "!dist/assets/**", ".next/**"], + "outputMode": "new-only" + }, // mocked test comment + "lint": { + "outputs": [], + "dependsOn": ["$MY_VAR"], + "cache": true, + "outputMode": "new-only" + }, + "dev": { + "cache": false, + "outputMode": "full" + }, + /* mocked test comment */ + "publish": { + "outputs": ["dist/**"], + "inputs": [ + /* + mocked test comment + */ + "build/**/*" + ], + "dependsOn": [ + /* mocked test comment */ "^publish", + "^build", + "build", + "admin#lint" + ], + "cache": false + } + }, + "globalDependencies": ["some-file", "../another-dir/**", "$GLOBAL_ENV_VAR"], + "globlaEnv": ["SOME_VAR", "ANOTHER_VAR"], + "experimentalGlobalPassThroughEnv": ["AWS_SECRET_KEY"], + "remoteCache": { + "teamId": "team_id", + "signature": true + } +} diff --git a/cli/internal/fs/testdata/invalid-env-1/turbo.json b/cli/internal/fs/testdata/invalid-env-1/turbo.json new file mode 100644 index 0000000..e4a6517 --- /dev/null +++ b/cli/internal/fs/testdata/invalid-env-1/turbo.json @@ -0,0 +1,8 @@ +{ + "pipeline": { + "task1": { + // all invalid value + "env": ["$A", "$B"] + } + } +} diff --git a/cli/internal/fs/testdata/invalid-env-2/turbo.json b/cli/internal/fs/testdata/invalid-env-2/turbo.json new file mode 100644 index 0000000..92eec96 --- /dev/null +++ b/cli/internal/fs/testdata/invalid-env-2/turbo.json @@ -0,0 +1,8 @@ +{ + "pipeline": { + "task1": { + // Mixed values + "env": ["$A", "B"] + } + } +} diff --git a/cli/internal/fs/testdata/invalid-global-env/turbo.json b/cli/internal/fs/testdata/invalid-global-env/turbo.json new file mode 100644 index 0000000..2ae9ff9 --- /dev/null +++ b/cli/internal/fs/testdata/invalid-global-env/turbo.json @@ -0,0 +1,11 @@ +{ + // Both global declarations with duplicates + "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"], + // some invalid values + "globalEnv": ["FOO", "BAZ", "$QUX"], + "pipeline": { + "task1": { + "dependsOn": ["$A"] + } + } +} diff --git a/cli/internal/fs/testdata/legacy-env/turbo.json b/cli/internal/fs/testdata/legacy-env/turbo.json new file mode 100644 index 0000000..6b082c4 --- /dev/null +++ b/cli/internal/fs/testdata/legacy-env/turbo.json @@ -0,0 +1,34 @@ +// mocked test comment +{ + // Both global declarations with duplicates and with + "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"], + "globalEnv": ["FOO", "BAZ", "QUX"], + "pipeline": { + // Only legacy declaration + "task1": { + "dependsOn": ["$A"] + }, + // Only new declaration + "task2": { + "env": ["A"] + }, + // Same var declared in both + "task3": { + "dependsOn": ["$A"], + "env": ["A"] + }, + // Different vars declared in both + "task4": { + "dependsOn": ["$A"], + "env": ["B"] + }, + + // some edge cases + "task6": { "env": ["A", "B", "C"], "dependsOn": ["$D", "$E", "$F"] }, + "task7": { "env": ["A", "B", "C"], "dependsOn": ["$A", "$B", "$C"] }, + "task8": { "env": ["A", "B", "C"], "dependsOn": ["A", "B", "C"] }, + "task9": { "env": [], "dependsOn": ["$A"] }, + "task10": { "env": ["A", "A"], "dependsOn": ["$A", "$A"] }, + "task11": { "env": ["A", "A"], "dependsOn": ["$B", "$B"] } + } +} diff --git a/cli/internal/fs/testdata/legacy-only/package.json b/cli/internal/fs/testdata/legacy-only/package.json new file mode 100644 index 0000000..03534b7 --- /dev/null +++ b/cli/internal/fs/testdata/legacy-only/package.json @@ -0,0 +1,7 @@ +{ + "turbo": { + "pipeline": { + "build": {} + } + } +} diff --git a/cli/internal/fs/turbo_json.go b/cli/internal/fs/turbo_json.go new file mode 100644 index 0000000..71ef29d --- /dev/null +++ b/cli/internal/fs/turbo_json.go @@ -0,0 +1,741 @@ +package fs + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/muhammadmuzzammil1998/jsonc" + "github.com/pkg/errors" + "github.com/vercel/turbo/cli/internal/turbopath" + "github.com/vercel/turbo/cli/internal/util" +) + +const ( + configFile = "turbo.json" + envPipelineDelimiter = "$" + topologicalPipelineDelimiter = "^" +) + +type rawTurboJSON struct { + // Global root filesystem dependencies + GlobalDependencies []string `json:"globalDependencies,omitempty"` + // Global env + GlobalEnv []string `json:"globalEnv,omitempty"` + + // Global passthrough env + GlobalPassthroughEnv []string `json:"experimentalGlobalPassThroughEnv,omitempty"` + + // Pipeline is a map of Turbo pipeline entries which define the task graph + // and cache behavior on a per task or per package-task basis. + Pipeline Pipeline `json:"pipeline"` + // Configuration options when interfacing with the remote cache + RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"` + + // Extends can be the name of another workspace + Extends []string `json:"extends,omitempty"` +} + +// pristineTurboJSON is used when marshaling a TurboJSON object into a turbo.json string +// Notably, it includes a PristinePipeline instead of the regular Pipeline. (i.e. TaskDefinition +// instead of BookkeepingTaskDefinition.) +type pristineTurboJSON struct { + GlobalDependencies []string `json:"globalDependencies,omitempty"` + GlobalEnv []string `json:"globalEnv,omitempty"` + GlobalPassthroughEnv []string `json:"experimentalGlobalPassThroughEnv,omitempty"` + Pipeline PristinePipeline `json:"pipeline"` + RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"` + Extends []string `json:"extends,omitempty"` +} + +// TurboJSON represents a turbo.json configuration file +type TurboJSON struct { + GlobalDeps []string + GlobalEnv []string + GlobalPassthroughEnv []string + Pipeline Pipeline + RemoteCacheOptions RemoteCacheOptions + + // A list of Workspace names + Extends []string +} + +// RemoteCacheOptions is a struct for deserializing .remoteCache of configFile +type RemoteCacheOptions struct { + TeamID string `json:"teamId,omitempty"` + Signature bool `json:"signature,omitempty"` +} + +// rawTaskWithDefaults exists to Marshal (i.e. turn a TaskDefinition into json). +// We use this for printing ResolvedTaskConfiguration, because we _want_ to show +// the user the default values for key they have not configured. +type rawTaskWithDefaults struct { + Outputs []string `json:"outputs"` + Cache *bool `json:"cache"` + DependsOn []string `json:"dependsOn"` + Inputs []string `json:"inputs"` + OutputMode util.TaskOutputMode `json:"outputMode"` + PassthroughEnv []string `json:"experimentalPassThroughEnv,omitempty"` + Env []string `json:"env"` + Persistent bool `json:"persistent"` +} + +// rawTask exists to Unmarshal from json. When fields are omitted, we _want_ +// them to be missing, so that we can distinguish missing from empty value. +type rawTask struct { + Outputs []string `json:"outputs,omitempty"` + Cache *bool `json:"cache,omitempty"` + DependsOn []string `json:"dependsOn,omitempty"` + Inputs []string `json:"inputs,omitempty"` + OutputMode *util.TaskOutputMode `json:"outputMode,omitempty"` + Env []string `json:"env,omitempty"` + PassthroughEnv []string `json:"experimentalPassthroughEnv,omitempty"` + Persistent *bool `json:"persistent,omitempty"` +} + +// taskDefinitionHashable exists as a definition for PristinePipeline, which is used down +// stream for calculating the global hash. We want to exclude experimental fields here +// because we don't want experimental fields to be part of the global hash. +type taskDefinitionHashable struct { + Outputs TaskOutputs + ShouldCache bool + EnvVarDependencies []string + TopologicalDependencies []string + TaskDependencies []string + Inputs []string + OutputMode util.TaskOutputMode + Persistent bool +} + +// taskDefinitionExperiments is a list of config fields in a task definition that are considered +// experimental. We keep these separated so we can compute a global hash without these. +type taskDefinitionExperiments struct { + PassthroughEnv []string +} + +// PristinePipeline is a map of task names to TaskDefinition or taskDefinitionHashable. +// Depending on whether any experimental fields are defined, we will use either struct. +// The purpose is to omit experimental fields when making a pristine version, so that +// it doesn't show up in --dry/--summarize output or affect the global hash. +type PristinePipeline map[string]interface{} + +// Pipeline is a struct for deserializing .pipeline in configFile +type Pipeline map[string]BookkeepingTaskDefinition + +// BookkeepingTaskDefinition holds the underlying TaskDefinition and some bookkeeping data +// about the TaskDefinition. This wrapper struct allows us to leave TaskDefinition untouched. +type BookkeepingTaskDefinition struct { + definedFields util.Set + experimentalFields util.Set + experimental taskDefinitionExperiments + TaskDefinition taskDefinitionHashable +} + +// TaskDefinition is a representation of the configFile pipeline for further computation. +type TaskDefinition struct { + Outputs TaskOutputs + ShouldCache bool + + // This field is custom-marshalled from rawTask.Env and rawTask.DependsOn + EnvVarDependencies []string + + // rawTask.PassthroughEnv + PassthroughEnv []string + + // TopologicalDependencies are tasks from package dependencies. + // E.g. "build" is a topological dependency in: + // dependsOn: ['^build']. + // This field is custom-marshalled from rawTask.DependsOn + TopologicalDependencies []string + + // TaskDependencies are anything that is not a topological dependency + // E.g. both something and //whatever are TaskDependencies in: + // dependsOn: ['something', '//whatever'] + // This field is custom-marshalled from rawTask.DependsOn + TaskDependencies []string + + // Inputs indicate the list of files this Task depends on. If any of those files change + // we can conclude that any cached outputs or logs for this Task should be invalidated. + Inputs []string + + // OutputMode determins how we should log the output. + OutputMode util.TaskOutputMode + + // Persistent indicates whether the Task is expected to exit or not + // Tasks marked Persistent do not exit (e.g. --watch mode or dev servers) + Persistent bool +} + +// GetTask returns a TaskDefinition based on the ID (package#task format) or name (e.g. "build") +func (pc Pipeline) GetTask(taskID string, taskName string) (*BookkeepingTaskDefinition, error) { + // first check for package-tasks + taskDefinition, ok := pc[taskID] + if !ok { + // then check for regular tasks + fallbackTaskDefinition, notcool := pc[taskName] + // if neither, then bail + if !notcool { + // Return an empty TaskDefinition + return nil, fmt.Errorf("Could not find task \"%s\" in pipeline", taskID) + } + + // override if we need to... + taskDefinition = fallbackTaskDefinition + } + + return &taskDefinition, nil +} + +// LoadTurboConfig loads, or optionally, synthesizes a TurboJSON instance +func LoadTurboConfig(dir turbopath.AbsoluteSystemPath, rootPackageJSON *PackageJSON, includeSynthesizedFromRootPackageJSON bool) (*TurboJSON, error) { + // If the root package.json stil has a `turbo` key, log a warning and remove it. + if rootPackageJSON.LegacyTurboConfig != nil { + log.Printf("[WARNING] \"turbo\" in package.json is no longer supported. Migrate to %s by running \"npx @turbo/codemod create-turbo-config\"\n", configFile) + rootPackageJSON.LegacyTurboConfig = nil + } + + var turboJSON *TurboJSON + turboFromFiles, err := readTurboConfig(dir.UntypedJoin(configFile)) + + if !includeSynthesizedFromRootPackageJSON && err != nil { + // If the file didn't exist, throw a custom error here instead of propagating + if errors.Is(err, os.ErrNotExist) { + return nil, errors.Wrap(err, fmt.Sprintf("Could not find %s. Follow directions at https://turbo.build/repo/docs to create one", configFile)) + + } + + // There was an error, and we don't have any chance of recovering + // because we aren't synthesizing anything + return nil, err + } else if !includeSynthesizedFromRootPackageJSON { + // We're not synthesizing anything and there was no error, we're done + return turboFromFiles, nil + } else if errors.Is(err, os.ErrNotExist) { + // turbo.json doesn't exist, but we're going try to synthesize something + turboJSON = &TurboJSON{ + Pipeline: make(Pipeline), + } + } else if err != nil { + // some other happened, we can't recover + return nil, err + } else { + // we're synthesizing, but we have a starting point + // Note: this will have to change to support task inference in a monorepo + // for now, we're going to error on any "root" tasks and turn non-root tasks into root tasks + pipeline := make(Pipeline) + for taskID, taskDefinition := range turboFromFiles.Pipeline { + if util.IsPackageTask(taskID) { + return nil, fmt.Errorf("Package tasks (#) are not allowed in single-package repositories: found %v", taskID) + } + pipeline[util.RootTaskID(taskID)] = taskDefinition + } + turboJSON = turboFromFiles + turboJSON.Pipeline = pipeline + } + + for scriptName := range rootPackageJSON.Scripts { + if !turboJSON.Pipeline.HasTask(scriptName) { + taskName := util.RootTaskID(scriptName) + // Explicitly set ShouldCache to false in this definition and add the bookkeeping fields + // so downstream we can pretend that it was set on purpose (as if read from a config file) + // rather than defaulting to the 0-value of a boolean field. + turboJSON.Pipeline[taskName] = BookkeepingTaskDefinition{ + definedFields: util.SetFromStrings([]string{"ShouldCache"}), + TaskDefinition: taskDefinitionHashable{ + ShouldCache: false, + }, + } + } + } + return turboJSON, nil +} + +// TurboJSONValidation is the signature for a validation function passed to Validate() +type TurboJSONValidation func(*TurboJSON) []error + +// Validate calls an array of validation functions on the TurboJSON struct. +// The validations can be customized by the caller. +func (tj *TurboJSON) Validate(validations []TurboJSONValidation) []error { + allErrors := []error{} + for _, validation := range validations { + errors := validation(tj) + allErrors = append(allErrors, errors...) + } + + return allErrors +} + +// TaskOutputs represents the patterns for including and excluding files from outputs +type TaskOutputs struct { + Inclusions []string + Exclusions []string +} + +// Sort contents of task outputs +func (to TaskOutputs) Sort() TaskOutputs { + var inclusions []string + var exclusions []string + copy(inclusions, to.Inclusions) + copy(exclusions, to.Exclusions) + sort.Strings(inclusions) + sort.Strings(exclusions) + return TaskOutputs{Inclusions: inclusions, Exclusions: exclusions} +} + +// readTurboConfig reads turbo.json from a provided path +func readTurboConfig(turboJSONPath turbopath.AbsoluteSystemPath) (*TurboJSON, error) { + // If the configFile exists, use that + if turboJSONPath.FileExists() { + turboJSON, err := readTurboJSON(turboJSONPath) + if err != nil { + return nil, fmt.Errorf("%s: %w", configFile, err) + } + + return turboJSON, nil + } + + // If there's no turbo.json, return an error. + return nil, os.ErrNotExist +} + +// readTurboJSON reads the configFile in to a struct +func readTurboJSON(path turbopath.AbsoluteSystemPath) (*TurboJSON, error) { + file, err := path.Open() + if err != nil { + return nil, err + } + var turboJSON *TurboJSON + data, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + err = jsonc.Unmarshal(data, &turboJSON) + + if err != nil { + return nil, err + } + + return turboJSON, nil +} + +// GetTaskDefinition returns a TaskDefinition from a serialized definition in configFile +func (pc Pipeline) GetTaskDefinition(taskID string) (TaskDefinition, bool) { + if entry, ok := pc[taskID]; ok { + return entry.GetTaskDefinition(), true + } + _, task := util.GetPackageTaskFromId(taskID) + entry, ok := pc[task] + return entry.GetTaskDefinition(), ok +} + +// HasTask returns true if the given task is defined in the pipeline, either directly or +// via a package task (`pkg#task`) +func (pc Pipeline) HasTask(task string) bool { + for key := range pc { + if key == task { + return true + } + if util.IsPackageTask(key) { + _, taskName := util.GetPackageTaskFromId(key) + if taskName == task { + return true + } + } + } + return false +} + +// Pristine returns a PristinePipeline, this is used for printing to console and pruning +func (pc Pipeline) Pristine() PristinePipeline { + pristine := PristinePipeline{} + for taskName, taskDef := range pc { + // If there are any experimental fields, we will include them with 0-values + // if there aren't, we will omit them entirely + if taskDef.hasExperimentalFields() { + pristine[taskName] = taskDef.GetTaskDefinition() // merges experimental fields in + } else { + pristine[taskName] = taskDef.TaskDefinition // has no experimental fields + } + } + return pristine +} + +// hasField checks the internal bookkeeping definedFields field to +// see whether a field was actually in the underlying turbo.json +// or whether it was initialized with its 0-value. +func (btd BookkeepingTaskDefinition) hasField(fieldName string) bool { + return btd.definedFields.Includes(fieldName) || btd.experimentalFields.Includes(fieldName) +} + +// hasExperimentalFields keeps track of whether any experimental fields were found +func (btd BookkeepingTaskDefinition) hasExperimentalFields() bool { + return len(btd.experimentalFields) > 0 +} + +// GetTaskDefinition gets a TaskDefinition by merging the experimental and non-experimental fields +// into a single representation to use downstream. +func (btd BookkeepingTaskDefinition) GetTaskDefinition() TaskDefinition { + return TaskDefinition{ + Outputs: btd.TaskDefinition.Outputs, + ShouldCache: btd.TaskDefinition.ShouldCache, + EnvVarDependencies: btd.TaskDefinition.EnvVarDependencies, + TopologicalDependencies: btd.TaskDefinition.TopologicalDependencies, + TaskDependencies: btd.TaskDefinition.TaskDependencies, + Inputs: btd.TaskDefinition.Inputs, + OutputMode: btd.TaskDefinition.OutputMode, + Persistent: btd.TaskDefinition.Persistent, + // From experimental fields + PassthroughEnv: btd.experimental.PassthroughEnv, + } +} + +// MergeTaskDefinitions accepts an array of BookkeepingTaskDefinitions and merges them into +// a single TaskDefinition. It uses the bookkeeping definedFields to determine which fields should +// be overwritten and when 0-values should be respected. +func MergeTaskDefinitions(taskDefinitions []BookkeepingTaskDefinition) (*TaskDefinition, error) { + // Start with an empty definition + mergedTaskDefinition := &TaskDefinition{} + + // Set the default, because the 0-value will be false, and if no turbo.jsons had + // this field set for this task, we want it to be true. + mergedTaskDefinition.ShouldCache = true + + // For each of the TaskDefinitions we know of, merge them in + for _, bookkeepingTaskDef := range taskDefinitions { + taskDef := bookkeepingTaskDef.GetTaskDefinition() + + if bookkeepingTaskDef.hasField("Outputs") { + mergedTaskDefinition.Outputs = taskDef.Outputs + } + + if bookkeepingTaskDef.hasField("ShouldCache") { + mergedTaskDefinition.ShouldCache = taskDef.ShouldCache + } + + if bookkeepingTaskDef.hasField("EnvVarDependencies") { + mergedTaskDefinition.EnvVarDependencies = taskDef.EnvVarDependencies + } + + if bookkeepingTaskDef.hasField("PassthroughEnv") { + mergedTaskDefinition.PassthroughEnv = taskDef.PassthroughEnv + } + + if bookkeepingTaskDef.hasField("DependsOn") { + mergedTaskDefinition.TopologicalDependencies = taskDef.TopologicalDependencies + } + + if bookkeepingTaskDef.hasField("DependsOn") { + mergedTaskDefinition.TaskDependencies = taskDef.TaskDependencies + } + + if bookkeepingTaskDef.hasField("Inputs") { + mergedTaskDefinition.Inputs = taskDef.Inputs + } + + if bookkeepingTaskDef.hasField("OutputMode") { + mergedTaskDefinition.OutputMode = taskDef.OutputMode + } + if bookkeepingTaskDef.hasField("Persistent") { + mergedTaskDefinition.Persistent = taskDef.Persistent + } + } + + return mergedTaskDefinition, nil +} + +// UnmarshalJSON deserializes a single task definition from +// turbo.json into a TaskDefinition struct +func (btd *BookkeepingTaskDefinition) UnmarshalJSON(data []byte) error { + task := rawTask{} + if err := json.Unmarshal(data, &task); err != nil { + return err + } + + btd.definedFields = util.Set{} + btd.experimentalFields = util.Set{} + + if task.Outputs != nil { + var inclusions []string + var exclusions []string + // Assign a bookkeeping field so we know that there really were + // outputs configured in the underlying config file. + btd.definedFields.Add("Outputs") + + for _, glob := range task.Outputs { + if strings.HasPrefix(glob, "!") { + if filepath.IsAbs(glob[1:]) { + log.Printf("[WARNING] Using an absolute path in \"outputs\" (%v) will not work and will be an error in a future version", glob) + } + exclusions = append(exclusions, glob[1:]) + } else { + if filepath.IsAbs(glob) { + log.Printf("[WARNING] Using an absolute path in \"outputs\" (%v) will not work and will be an error in a future version", glob) + } + inclusions = append(inclusions, glob) + } + } + + btd.TaskDefinition.Outputs = TaskOutputs{ + Inclusions: inclusions, + Exclusions: exclusions, + } + + sort.Strings(btd.TaskDefinition.Outputs.Inclusions) + sort.Strings(btd.TaskDefinition.Outputs.Exclusions) + } + + if task.Cache == nil { + btd.TaskDefinition.ShouldCache = true + } else { + btd.definedFields.Add("ShouldCache") + btd.TaskDefinition.ShouldCache = *task.Cache + } + + envVarDependencies := make(util.Set) + envVarPassthroughs := make(util.Set) + + btd.TaskDefinition.TopologicalDependencies = []string{} // TODO @mehulkar: this should be a set + btd.TaskDefinition.TaskDependencies = []string{} // TODO @mehulkar: this should be a set + + // If there was a dependsOn field, add the bookkeeping + // we don't care what's in the field, just that it was there + // We'll use this marker to overwrite while merging TaskDefinitions. + if task.DependsOn != nil { + btd.definedFields.Add("DependsOn") + } + + for _, dependency := range task.DependsOn { + if strings.HasPrefix(dependency, envPipelineDelimiter) { + log.Printf("[DEPRECATED] Declaring an environment variable in \"dependsOn\" is deprecated, found %s. Use the \"env\" key or use `npx @turbo/codemod migrate-env-var-dependencies`.\n", dependency) + envVarDependencies.Add(strings.TrimPrefix(dependency, envPipelineDelimiter)) + } else if strings.HasPrefix(dependency, topologicalPipelineDelimiter) { + // Note: This will get assigned multiple times in the loop, but we only care that it's true + btd.TaskDefinition.TopologicalDependencies = append(btd.TaskDefinition.TopologicalDependencies, strings.TrimPrefix(dependency, topologicalPipelineDelimiter)) + } else { + btd.TaskDefinition.TaskDependencies = append(btd.TaskDefinition.TaskDependencies, dependency) + } + } + + sort.Strings(btd.TaskDefinition.TaskDependencies) + sort.Strings(btd.TaskDefinition.TopologicalDependencies) + + // Append env key into EnvVarDependencies + if task.Env != nil { + btd.definedFields.Add("EnvVarDependencies") + if err := gatherEnvVars(task.Env, "env", &envVarDependencies); err != nil { + return err + } + } + + btd.TaskDefinition.EnvVarDependencies = envVarDependencies.UnsafeListOfStrings() + + sort.Strings(btd.TaskDefinition.EnvVarDependencies) + + if task.PassthroughEnv != nil { + btd.experimentalFields.Add("PassthroughEnv") + if err := gatherEnvVars(task.PassthroughEnv, "passthrougEnv", &envVarPassthroughs); err != nil { + return err + } + } + + btd.experimental.PassthroughEnv = envVarPassthroughs.UnsafeListOfStrings() + sort.Strings(btd.experimental.PassthroughEnv) + + if task.Inputs != nil { + // Note that we don't require Inputs to be sorted, we're going to + // hash the resulting files and sort that instead + btd.definedFields.Add("Inputs") + // TODO: during rust port, this should be moved to a post-parse validation step + for _, input := range task.Inputs { + if filepath.IsAbs(input) { + log.Printf("[WARNING] Using an absolute path in \"inputs\" (%v) will not work and will be an error in a future version", input) + } + } + btd.TaskDefinition.Inputs = task.Inputs + } + + if task.OutputMode != nil { + btd.definedFields.Add("OutputMode") + btd.TaskDefinition.OutputMode = *task.OutputMode + } + + if task.Persistent != nil { + btd.definedFields.Add("Persistent") + btd.TaskDefinition.Persistent = *task.Persistent + } else { + btd.TaskDefinition.Persistent = false + } + return nil +} + +// MarshalJSON serializes taskDefinitionHashable struct into json +func (c taskDefinitionHashable) MarshalJSON() ([]byte, error) { + task := makeRawTask( + c.Persistent, + c.ShouldCache, + c.OutputMode, + c.Inputs, + c.Outputs, + c.EnvVarDependencies, + c.TaskDependencies, + c.TopologicalDependencies, + ) + return json.Marshal(task) +} + +// MarshalJSON serializes TaskDefinition struct into json +func (c TaskDefinition) MarshalJSON() ([]byte, error) { + task := makeRawTask( + c.Persistent, + c.ShouldCache, + c.OutputMode, + c.Inputs, + c.Outputs, + c.EnvVarDependencies, + c.TaskDependencies, + c.TopologicalDependencies, + ) + + if len(c.PassthroughEnv) > 0 { + task.PassthroughEnv = append(task.PassthroughEnv, c.PassthroughEnv...) + } + sort.Strings(task.PassthroughEnv) + + return json.Marshal(task) +} + +// UnmarshalJSON deserializes the contents of turbo.json into a TurboJSON struct +func (c *TurboJSON) UnmarshalJSON(data []byte) error { + raw := &rawTurboJSON{} + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + envVarDependencies := make(util.Set) + envVarPassthroughs := make(util.Set) + globalFileDependencies := make(util.Set) + + if err := gatherEnvVars(raw.GlobalEnv, "globalEnv", &envVarDependencies); err != nil { + return err + } + if err := gatherEnvVars(raw.GlobalPassthroughEnv, "experimentalGlobalPassThroughEnv", &envVarPassthroughs); err != nil { + return err + } + + // TODO: In the rust port, warnings should be refactored to a post-parse validation step + for _, value := range raw.GlobalDependencies { + if strings.HasPrefix(value, envPipelineDelimiter) { + log.Printf("[DEPRECATED] Declaring an environment variable in \"globalDependencies\" is deprecated, found %s. Use the \"globalEnv\" key or use `npx @turbo/codemod migrate-env-var-dependencies`.\n", value) + envVarDependencies.Add(strings.TrimPrefix(value, envPipelineDelimiter)) + } else { + if filepath.IsAbs(value) { + log.Printf("[WARNING] Using an absolute path in \"globalDependencies\" (%v) will not work and will be an error in a future version", value) + } + globalFileDependencies.Add(value) + } + } + + // turn the set into an array and assign to the TurboJSON struct fields. + c.GlobalEnv = envVarDependencies.UnsafeListOfStrings() + sort.Strings(c.GlobalEnv) + + if raw.GlobalPassthroughEnv != nil { + c.GlobalPassthroughEnv = envVarPassthroughs.UnsafeListOfStrings() + sort.Strings(c.GlobalPassthroughEnv) + } + + c.GlobalDeps = globalFileDependencies.UnsafeListOfStrings() + sort.Strings(c.GlobalDeps) + + // copy these over, we don't need any changes here. + c.Pipeline = raw.Pipeline + c.RemoteCacheOptions = raw.RemoteCacheOptions + c.Extends = raw.Extends + + return nil +} + +// MarshalJSON converts a TurboJSON into the equivalent json object in bytes +// note: we go via rawTurboJSON so that the output format is correct. +// This is used by `turbo prune` to generate a pruned turbo.json +// and also by --summarize & --dry=json to serialize the known config +// into something we can print to screen +func (c *TurboJSON) MarshalJSON() ([]byte, error) { + raw := pristineTurboJSON{} + raw.GlobalDependencies = c.GlobalDeps + raw.GlobalEnv = c.GlobalEnv + raw.GlobalPassthroughEnv = c.GlobalPassthroughEnv + raw.Pipeline = c.Pipeline.Pristine() + raw.RemoteCacheOptions = c.RemoteCacheOptions + + return json.Marshal(&raw) +} + +func makeRawTask(persistent bool, shouldCache bool, outputMode util.TaskOutputMode, inputs []string, outputs TaskOutputs, envVarDependencies []string, taskDependencies []string, topologicalDependencies []string) *rawTaskWithDefaults { + // Initialize with empty arrays, so we get empty arrays serialized into JSON + task := &rawTaskWithDefaults{ + Outputs: []string{}, + Inputs: []string{}, + Env: []string{}, + PassthroughEnv: []string{}, + DependsOn: []string{}, + } + + task.Persistent = persistent + task.Cache = &shouldCache + task.OutputMode = outputMode + + if len(inputs) > 0 { + task.Inputs = inputs + } + + if len(envVarDependencies) > 0 { + task.Env = append(task.Env, envVarDependencies...) + } + + if len(outputs.Inclusions) > 0 { + task.Outputs = append(task.Outputs, outputs.Inclusions...) + } + + for _, i := range outputs.Exclusions { + task.Outputs = append(task.Outputs, "!"+i) + } + + if len(taskDependencies) > 0 { + task.DependsOn = append(task.DependsOn, taskDependencies...) + } + + for _, i := range topologicalDependencies { + task.DependsOn = append(task.DependsOn, "^"+i) + } + + // These _should_ already be sorted when the TaskDefinition struct was unmarshaled, + // but we want to ensure they're sorted on the way out also, just in case something + // in the middle mutates the items. + sort.Strings(task.DependsOn) + sort.Strings(task.Outputs) + sort.Strings(task.Env) + sort.Strings(task.Inputs) + return task +} + +// gatherEnvVars puts env vars into the provided set as long as they don't have an invalid value. +func gatherEnvVars(vars []string, key string, into *util.Set) error { + for _, value := range vars { + if strings.HasPrefix(value, envPipelineDelimiter) { + // Hard error to help people specify this correctly during migration. + // TODO: Remove this error after we have run summary. + return fmt.Errorf("You specified \"%s\" in the \"%s\" key. You should not prefix your environment variables with \"%s\"", value, key, envPipelineDelimiter) + } + + into.Add(value) + } + + return nil +} diff --git a/cli/internal/fs/turbo_json_test.go b/cli/internal/fs/turbo_json_test.go new file mode 100644 index 0000000..1d384d5 --- /dev/null +++ b/cli/internal/fs/turbo_json_test.go @@ -0,0 +1,277 @@ +package fs + +import ( + "os" + "reflect" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vercel/turbo/cli/internal/turbopath" + "github.com/vercel/turbo/cli/internal/util" + "gotest.tools/v3/assert/cmp" +) + +func assertIsSorted(t *testing.T, arr []string, msg string) { + t.Helper() + if arr == nil { + return + } + + copied := make([]string, len(arr)) + copy(copied, arr) + sort.Strings(copied) + if !reflect.DeepEqual(arr, copied) { + t.Errorf("Expected sorted, got %v: %v", arr, msg) + } +} + +func Test_ReadTurboConfig(t *testing.T) { + testDir := getTestDir(t, "correct") + turboJSON, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) + + if turboJSONReadErr != nil { + t.Fatalf("invalid parse: %#v", turboJSONReadErr) + } + + assert.EqualValues(t, []string{"AWS_SECRET_KEY"}, turboJSON.GlobalPassthroughEnv) + + pipelineExpected := map[string]BookkeepingTaskDefinition{ + "build": { + definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "DependsOn"}), + experimentalFields: util.SetFromStrings([]string{"PassthroughEnv"}), + experimental: taskDefinitionExperiments{ + PassthroughEnv: []string{"GITHUB_TOKEN"}, + }, + TaskDefinition: taskDefinitionHashable{ + Outputs: TaskOutputs{Inclusions: []string{".next/**", "dist/**"}, Exclusions: []string{"dist/assets/**"}}, + TopologicalDependencies: []string{"build"}, + EnvVarDependencies: []string{}, + TaskDependencies: []string{}, + ShouldCache: true, + OutputMode: util.NewTaskOutput, + }, + }, + "lint": { + definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "ShouldCache", "DependsOn"}), + experimentalFields: util.SetFromStrings([]string{}), + experimental: taskDefinitionExperiments{ + PassthroughEnv: []string{}, + }, + TaskDefinition: taskDefinitionHashable{ + Outputs: TaskOutputs{}, + TopologicalDependencies: []string{}, + EnvVarDependencies: []string{"MY_VAR"}, + TaskDependencies: []string{}, + ShouldCache: true, + OutputMode: util.NewTaskOutput, + }, + }, + "dev": { + definedFields: util.SetFromStrings([]string{"OutputMode", "ShouldCache"}), + experimentalFields: util.SetFromStrings([]string{}), + experimental: taskDefinitionExperiments{ + PassthroughEnv: []string{}, + }, + TaskDefinition: taskDefinitionHashable{ + Outputs: TaskOutputs{}, + TopologicalDependencies: []string{}, + EnvVarDependencies: []string{}, + TaskDependencies: []string{}, + ShouldCache: false, + OutputMode: util.FullTaskOutput, + }, + }, + "publish": { + definedFields: util.SetFromStrings([]string{"Inputs", "Outputs", "DependsOn", "ShouldCache"}), + experimentalFields: util.SetFromStrings([]string{}), + experimental: taskDefinitionExperiments{ + PassthroughEnv: []string{}, + }, + TaskDefinition: taskDefinitionHashable{ + Outputs: TaskOutputs{Inclusions: []string{"dist/**"}}, + TopologicalDependencies: []string{"build", "publish"}, + EnvVarDependencies: []string{}, + TaskDependencies: []string{"admin#lint", "build"}, + ShouldCache: false, + Inputs: []string{"build/**/*"}, + OutputMode: util.FullTaskOutput, + }, + }, + } + + validateOutput(t, turboJSON, pipelineExpected) + remoteCacheOptionsExpected := RemoteCacheOptions{"team_id", true} + assert.EqualValues(t, remoteCacheOptionsExpected, turboJSON.RemoteCacheOptions) +} + +func Test_LoadTurboConfig_Legacy(t *testing.T) { + testDir := getTestDir(t, "legacy-only") + packageJSONPath := testDir.UntypedJoin("package.json") + rootPackageJSON, pkgJSONReadErr := ReadPackageJSON(packageJSONPath) + + if pkgJSONReadErr != nil { + t.Fatalf("invalid parse: %#v", pkgJSONReadErr) + } + + _, turboJSONReadErr := LoadTurboConfig(testDir, rootPackageJSON, false) + expectedErrorMsg := "Could not find turbo.json. Follow directions at https://turbo.build/repo/docs to create one: file does not exist" + assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) +} + +func Test_LoadTurboConfig_BothCorrectAndLegacy(t *testing.T) { + testDir := getTestDir(t, "both") + + packageJSONPath := testDir.UntypedJoin("package.json") + rootPackageJSON, pkgJSONReadErr := ReadPackageJSON(packageJSONPath) + + if pkgJSONReadErr != nil { + t.Fatalf("invalid parse: %#v", pkgJSONReadErr) + } + + turboJSON, turboJSONReadErr := LoadTurboConfig(testDir, rootPackageJSON, false) + + if turboJSONReadErr != nil { + t.Fatalf("invalid parse: %#v", turboJSONReadErr) + } + + pipelineExpected := map[string]BookkeepingTaskDefinition{ + "build": { + definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "DependsOn"}), + experimentalFields: util.SetFromStrings([]string{}), + experimental: taskDefinitionExperiments{ + PassthroughEnv: []string{}, + }, + TaskDefinition: taskDefinitionHashable{ + Outputs: TaskOutputs{Inclusions: []string{".next/**", "dist/**"}, Exclusions: []string{"dist/assets/**"}}, + TopologicalDependencies: []string{"build"}, + EnvVarDependencies: []string{}, + TaskDependencies: []string{}, + ShouldCache: true, + OutputMode: util.NewTaskOutput, + }, + }, + } + + validateOutput(t, turboJSON, pipelineExpected) + + remoteCacheOptionsExpected := RemoteCacheOptions{"team_id", true} + assert.EqualValues(t, remoteCacheOptionsExpected, turboJSON.RemoteCacheOptions) + assert.Equal(t, rootPackageJSON.LegacyTurboConfig == nil, true) +} + +func Test_ReadTurboConfig_InvalidEnvDeclarations1(t *testing.T) { + testDir := getTestDir(t, "invalid-env-1") + _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) + + expectedErrorMsg := "turbo.json: You specified \"$A\" in the \"env\" key. You should not prefix your environment variables with \"$\"" + assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) +} + +func Test_ReadTurboConfig_InvalidEnvDeclarations2(t *testing.T) { + testDir := getTestDir(t, "invalid-env-2") + _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) + expectedErrorMsg := "turbo.json: You specified \"$A\" in the \"env\" key. You should not prefix your environment variables with \"$\"" + assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) +} + +func Test_ReadTurboConfig_InvalidGlobalEnvDeclarations(t *testing.T) { + testDir := getTestDir(t, "invalid-global-env") + _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) + expectedErrorMsg := "turbo.json: You specified \"$QUX\" in the \"globalEnv\" key. You should not prefix your environment variables with \"$\"" + assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr) +} + +func Test_ReadTurboConfig_EnvDeclarations(t *testing.T) { + testDir := getTestDir(t, "legacy-env") + turboJSON, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json")) + + if turboJSONReadErr != nil { + t.Fatalf("invalid parse: %#v", turboJSONReadErr) + } + + pipeline := turboJSON.Pipeline + assert.EqualValues(t, pipeline["task1"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) + assert.EqualValues(t, pipeline["task2"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) + assert.EqualValues(t, pipeline["task3"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) + assert.EqualValues(t, pipeline["task4"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B"})) + assert.EqualValues(t, pipeline["task6"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C", "D", "E", "F"})) + assert.EqualValues(t, pipeline["task7"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C"})) + assert.EqualValues(t, pipeline["task8"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C"})) + assert.EqualValues(t, pipeline["task9"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) + assert.EqualValues(t, pipeline["task10"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"})) + assert.EqualValues(t, pipeline["task11"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B"})) + + // check global env vars also + assert.EqualValues(t, sortedArray([]string{"FOO", "BAR", "BAZ", "QUX"}), sortedArray(turboJSON.GlobalEnv)) + assert.EqualValues(t, sortedArray([]string{"somefile.txt"}), sortedArray(turboJSON.GlobalDeps)) +} + +func Test_TaskOutputsSort(t *testing.T) { + inclusions := []string{"foo/**", "bar"} + exclusions := []string{"special-file", ".hidden/**"} + taskOutputs := TaskOutputs{Inclusions: inclusions, Exclusions: exclusions} + sortedOutputs := taskOutputs.Sort() + assertIsSorted(t, sortedOutputs.Inclusions, "Inclusions") + assertIsSorted(t, sortedOutputs.Exclusions, "Exclusions") + assert.False(t, cmp.DeepEqual(taskOutputs, sortedOutputs)().Success()) +} + +// Helpers +func validateOutput(t *testing.T, turboJSON *TurboJSON, expectedPipeline Pipeline) { + t.Helper() + assertIsSorted(t, turboJSON.GlobalDeps, "Global Deps") + assertIsSorted(t, turboJSON.GlobalEnv, "Global Env") + validatePipeline(t, turboJSON.Pipeline, expectedPipeline) +} + +func validatePipeline(t *testing.T, actual Pipeline, expected Pipeline) { + t.Helper() + // check top level keys + if len(actual) != len(expected) { + expectedKeys := []string{} + for k := range expected { + expectedKeys = append(expectedKeys, k) + } + actualKeys := []string{} + for k := range actual { + actualKeys = append(actualKeys, k) + } + t.Errorf("pipeline tasks mismatch. got %v, want %v", strings.Join(actualKeys, ","), strings.Join(expectedKeys, ",")) + } + + // check individual task definitions + for taskName, expectedTaskDefinition := range expected { + bookkeepingTaskDef, ok := actual[taskName] + if !ok { + t.Errorf("missing expected task: %v", taskName) + } + actualTaskDefinition := bookkeepingTaskDef.GetTaskDefinition() + assertIsSorted(t, actualTaskDefinition.Outputs.Inclusions, "Task output inclusions") + assertIsSorted(t, actualTaskDefinition.Outputs.Exclusions, "Task output exclusions") + assertIsSorted(t, actualTaskDefinition.EnvVarDependencies, "Task env vars") + assertIsSorted(t, actualTaskDefinition.PassthroughEnv, "Task env vars") + assertIsSorted(t, actualTaskDefinition.TopologicalDependencies, "Topo deps") + assertIsSorted(t, actualTaskDefinition.TaskDependencies, "Task deps") + assert.EqualValuesf(t, expectedTaskDefinition, bookkeepingTaskDef, "task definition mismatch for %v", taskName) + } +} + +func getTestDir(t *testing.T, testName string) turbopath.AbsoluteSystemPath { + defaultCwd, err := os.Getwd() + if err != nil { + t.Errorf("failed to get cwd: %v", err) + } + cwd, err := CheckedToAbsoluteSystemPath(defaultCwd) + if err != nil { + t.Fatalf("cwd is not an absolute directory %v: %v", defaultCwd, err) + } + + return cwd.UntypedJoin("testdata", testName) +} + +func sortedArray(arr []string) []string { + sort.Strings(arr) + return arr +} -- cgit v1.2.3-70-g09d2