aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/cli/internal/fs
diff options
context:
space:
mode:
author简律纯 <hsiangnianian@outlook.com>2023-04-28 01:36:55 +0800
committer简律纯 <hsiangnianian@outlook.com>2023-04-28 01:36:55 +0800
commitfc8c5fdce62fb229202659408798a7b6c98f6e8b (patch)
tree7554f80e50de4af6fd255afa7c21bcdd58a7af34 /cli/internal/fs
parentdd84b9d64fb98746a230cd24233ff50a562c39c9 (diff)
downloadHydroRoll-fc8c5fdce62fb229202659408798a7b6c98f6e8b.tar.gz
HydroRoll-fc8c5fdce62fb229202659408798a7b6c98f6e8b.zip
Diffstat (limited to 'cli/internal/fs')
-rw-r--r--cli/internal/fs/copy_file.go81
-rw-r--r--cli/internal/fs/copy_file_test.go198
-rw-r--r--cli/internal/fs/fs.go191
-rw-r--r--cli/internal/fs/fs_test.go60
-rw-r--r--cli/internal/fs/fs_windows_test.go18
-rw-r--r--cli/internal/fs/get_turbo_data_dir_go.go16
-rw-r--r--cli/internal/fs/get_turbo_data_dir_rust.go16
-rw-r--r--cli/internal/fs/hash.go61
-rw-r--r--cli/internal/fs/hash_test.go53
-rw-r--r--cli/internal/fs/lstat.go74
-rw-r--r--cli/internal/fs/package_json.go142
-rw-r--r--cli/internal/fs/package_json_test.go174
-rw-r--r--cli/internal/fs/path.go113
-rw-r--r--cli/internal/fs/testdata/both/package.json7
-rw-r--r--cli/internal/fs/testdata/both/turbo.json18
-rw-r--r--cli/internal/fs/testdata/correct/turbo.json49
-rw-r--r--cli/internal/fs/testdata/invalid-env-1/turbo.json8
-rw-r--r--cli/internal/fs/testdata/invalid-env-2/turbo.json8
-rw-r--r--cli/internal/fs/testdata/invalid-global-env/turbo.json11
-rw-r--r--cli/internal/fs/testdata/legacy-env/turbo.json34
-rw-r--r--cli/internal/fs/testdata/legacy-only/package.json7
-rw-r--r--cli/internal/fs/turbo_json.go741
-rw-r--r--cli/internal/fs/turbo_json_test.go277
23 files changed, 0 insertions, 2357 deletions
diff --git a/cli/internal/fs/copy_file.go b/cli/internal/fs/copy_file.go
deleted file mode 100644
index e7619de..0000000
--- a/cli/internal/fs/copy_file.go
+++ /dev/null
@@ -1,81 +0,0 @@
-// Adapted from https://github.com/thought-machine/please
-// Copyright Thought Machine, Inc. or its affiliates. All Rights Reserved.
-// SPDX-License-Identifier: Apache-2.0
-package fs
-
-import (
- "errors"
- "os"
- "path/filepath"
-
- "github.com/karrick/godirwalk"
-)
-
-// RecursiveCopy copies either a single file or a directory.
-// 'mode' is the mode of the destination file.
-func RecursiveCopy(from string, to string) error {
- // Verified all callers are passing in absolute paths for from (and to)
- statedFrom := LstatCachedFile{Path: UnsafeToAbsoluteSystemPath(from)}
- fromType, err := statedFrom.GetType()
- if err != nil {
- return err
- }
-
- if fromType.IsDir() {
- return WalkMode(statedFrom.Path.ToStringDuringMigration(), func(name string, isDir bool, fileType os.FileMode) error {
- dest := filepath.Join(to, name[len(statedFrom.Path.ToString()):])
- // name is absolute, (originates from godirwalk)
- src := LstatCachedFile{Path: UnsafeToAbsoluteSystemPath(name), fileType: &fileType}
- if isDir {
- mode, err := src.GetMode()
- if err != nil {
- return err
- }
- return os.MkdirAll(dest, mode)
- }
- return CopyFile(&src, dest)
- })
- }
- return CopyFile(&statedFrom, to)
-}
-
-// Walk implements an equivalent to filepath.Walk.
-// It's implemented over github.com/karrick/godirwalk but the provided interface doesn't use that
-// to make it a little easier to handle.
-func Walk(rootPath string, callback func(name string, isDir bool) error) error {
- return WalkMode(rootPath, func(name string, isDir bool, mode os.FileMode) error {
- return callback(name, isDir)
- })
-}
-
-// WalkMode is like Walk but the callback receives an additional type specifying the file mode type.
-// N.B. This only includes the bits of the mode that determine the mode type, not the permissions.
-func WalkMode(rootPath string, callback func(name string, isDir bool, mode os.FileMode) error) error {
- return godirwalk.Walk(rootPath, &godirwalk.Options{
- Callback: func(name string, info *godirwalk.Dirent) error {
- // currently we support symlinked files, but not symlinked directories:
- // For copying, we Mkdir and bail if we encounter a symlink to a directoy
- // For finding packages, we enumerate the symlink, but don't follow inside
- isDir, err := info.IsDirOrSymlinkToDir()
- if err != nil {
- pathErr := &os.PathError{}
- if errors.As(err, &pathErr) {
- // If we have a broken link, skip this entry
- return godirwalk.SkipThis
- }
- return err
- }
- return callback(name, isDir, info.ModeType())
- },
- ErrorCallback: func(pathname string, err error) godirwalk.ErrorAction {
- pathErr := &os.PathError{}
- if errors.As(err, &pathErr) {
- return godirwalk.SkipNode
- }
- return godirwalk.Halt
- },
- Unsorted: true,
- AllowNonDirectory: true,
- FollowSymbolicLinks: false,
- })
-}
diff --git a/cli/internal/fs/copy_file_test.go b/cli/internal/fs/copy_file_test.go
deleted file mode 100644
index 6a61576..0000000
--- a/cli/internal/fs/copy_file_test.go
+++ /dev/null
@@ -1,198 +0,0 @@
-package fs
-
-import (
- "errors"
- "io/ioutil"
- "os"
- "path/filepath"
- "testing"
-
- "github.com/vercel/turbo/cli/internal/turbopath"
- "gotest.tools/v3/assert"
- "gotest.tools/v3/fs"
-)
-
-func TestCopyFile(t *testing.T) {
- srcTmpDir := turbopath.AbsoluteSystemPath(t.TempDir())
- destTmpDir := turbopath.AbsoluteSystemPath(t.TempDir())
- srcFilePath := srcTmpDir.UntypedJoin("src")
- destFilePath := destTmpDir.UntypedJoin("dest")
- from := &LstatCachedFile{Path: srcFilePath}
-
- // The src file doesn't exist, will error.
- err := CopyFile(from, destFilePath.ToString())
- pathErr := &os.PathError{}
- if !errors.As(err, &pathErr) {
- t.Errorf("got %v, want PathError", err)
- }
-
- // Create the src file.
- srcFile, err := srcFilePath.Create()
- assert.NilError(t, err, "Create")
- _, err = srcFile.WriteString("src")
- assert.NilError(t, err, "WriteString")
- assert.NilError(t, srcFile.Close(), "Close")
-
- // Copy the src to the dest.
- err = CopyFile(from, destFilePath.ToString())
- assert.NilError(t, err, "src exists dest does not, should not error.")
-
- // Now test for symlinks.
- symlinkSrcDir := turbopath.AbsoluteSystemPath(t.TempDir())
- symlinkTargetDir := turbopath.AbsoluteSystemPath(t.TempDir())
- symlinkDestDir := turbopath.AbsoluteSystemPath(t.TempDir())
- symlinkSrcPath := symlinkSrcDir.UntypedJoin("symlink")
- symlinkTargetPath := symlinkTargetDir.UntypedJoin("target")
- symlinkDestPath := symlinkDestDir.UntypedJoin("dest")
- fromSymlink := &LstatCachedFile{Path: symlinkSrcPath}
-
- // Create the symlink target.
- symlinkTargetFile, err := symlinkTargetPath.Create()
- assert.NilError(t, err, "Create")
- _, err = symlinkTargetFile.WriteString("Target")
- assert.NilError(t, err, "WriteString")
- assert.NilError(t, symlinkTargetFile.Close(), "Close")
-
- // Link things up.
- err = symlinkSrcPath.Symlink(symlinkTargetPath.ToString())
- assert.NilError(t, err, "Symlink")
-
- // Run the test.
- err = CopyFile(fromSymlink, symlinkDestPath.ToString())
- assert.NilError(t, err, "Copying a valid symlink does not error.")
-
- // Break the symlink.
- err = symlinkTargetPath.Remove()
- assert.NilError(t, err, "breaking the symlink")
-
- // Remove the existing copy.
- err = symlinkDestPath.Remove()
- assert.NilError(t, err, "existing copy is removed")
-
- // Try copying the now-broken symlink.
- err = CopyFile(fromSymlink, symlinkDestPath.ToString())
- assert.NilError(t, err, "CopyFile")
-
- // Confirm that it copied
- target, err := symlinkDestPath.Readlink()
- assert.NilError(t, err, "Readlink")
- assert.Equal(t, target, symlinkTargetPath.ToString())
-}
-
-func TestCopyOrLinkFileWithPerms(t *testing.T) {
- // Directory layout:
- //
- // <src>/
- // foo
- readonlyMode := os.FileMode(0444)
- srcDir := turbopath.AbsoluteSystemPath(t.TempDir())
- dstDir := turbopath.AbsoluteSystemPath(t.TempDir())
- srcFilePath := srcDir.UntypedJoin("src")
- dstFilePath := dstDir.UntypedJoin("dst")
- srcFile, err := srcFilePath.Create()
- defer func() { _ = srcFile.Close() }()
- assert.NilError(t, err, "Create")
- err = srcFile.Chmod(readonlyMode)
- assert.NilError(t, err, "Chmod")
- err = CopyFile(&LstatCachedFile{Path: srcFilePath}, dstFilePath.ToStringDuringMigration())
- assert.NilError(t, err, "CopyOrLinkFile")
- info, err := dstFilePath.Lstat()
- assert.NilError(t, err, "Lstat")
- assert.Equal(t, info.Mode(), readonlyMode, "expected dest to have matching permissions")
-}
-
-func TestRecursiveCopy(t *testing.T) {
- // Directory layout:
- //
- // <src>/
- // b
- // child/
- // a
- // link -> ../b
- // broken -> missing
- // circle -> ../child
- src := fs.NewDir(t, "recursive-copy-or-link")
- dst := fs.NewDir(t, "recursive-copy-or-link-dist")
- childDir := filepath.Join(src.Path(), "child")
- err := os.Mkdir(childDir, os.ModeDir|0777)
- assert.NilError(t, err, "Mkdir")
- aPath := filepath.Join(childDir, "a")
- aFile, err := os.Create(aPath)
- assert.NilError(t, err, "Create")
- _, err = aFile.WriteString("hello")
- assert.NilError(t, err, "WriteString")
- assert.NilError(t, aFile.Close(), "Close")
-
- bPath := filepath.Join(src.Path(), "b")
- bFile, err := os.Create(bPath)
- assert.NilError(t, err, "Create")
- _, err = bFile.WriteString("bFile")
- assert.NilError(t, err, "WriteString")
- assert.NilError(t, bFile.Close(), "Close")
-
- srcLinkPath := filepath.Join(childDir, "link")
- assert.NilError(t, os.Symlink(filepath.FromSlash("../b"), srcLinkPath), "Symlink")
-
- srcBrokenLinkPath := filepath.Join(childDir, "broken")
- assert.NilError(t, os.Symlink("missing", srcBrokenLinkPath), "Symlink")
- circlePath := filepath.Join(childDir, "circle")
- assert.NilError(t, os.Symlink(filepath.FromSlash("../child"), circlePath), "Symlink")
-
- err = RecursiveCopy(src.Path(), dst.Path())
- assert.NilError(t, err, "RecursiveCopy")
- // For ensure multiple times copy will not broken
- err = RecursiveCopy(src.Path(), dst.Path())
- assert.NilError(t, err, "RecursiveCopy")
-
- dstChildDir := filepath.Join(dst.Path(), "child")
- assertDirMatches(t, childDir, dstChildDir)
- dstAPath := filepath.Join(dst.Path(), "child", "a")
- assertFileMatches(t, aPath, dstAPath)
- dstBPath := filepath.Join(dst.Path(), "b")
- assertFileMatches(t, bPath, dstBPath)
- dstLinkPath := filepath.Join(dst.Path(), "child", "link")
- dstLinkDest, err := os.Readlink(dstLinkPath)
- assert.NilError(t, err, "Readlink")
- expectedLinkDest := filepath.FromSlash("../b")
- if dstLinkDest != expectedLinkDest {
- t.Errorf("Readlink got %v, want %v", dstLinkDest, expectedLinkDest)
- }
- dstBrokenLinkPath := filepath.Join(dst.Path(), "child", "broken")
- brokenLinkExists := PathExists(dstBrokenLinkPath)
- if brokenLinkExists {
- t.Errorf("We cached a broken link at %v", dstBrokenLinkPath)
- }
- // Currently, we convert symlink-to-directory to empty-directory
- // This is very likely not ideal behavior, but leaving this test here to verify
- // that it is what we expect at this point in time.
- dstCirclePath := filepath.Join(dst.Path(), "child", "circle")
- circleStat, err := os.Lstat(dstCirclePath)
- assert.NilError(t, err, "Lstat")
- assert.Equal(t, circleStat.IsDir(), true)
- entries, err := os.ReadDir(dstCirclePath)
- assert.NilError(t, err, "ReadDir")
- assert.Equal(t, len(entries), 0)
-}
-
-func assertFileMatches(t *testing.T, orig string, copy string) {
- t.Helper()
- origBytes, err := ioutil.ReadFile(orig)
- assert.NilError(t, err, "ReadFile")
- copyBytes, err := ioutil.ReadFile(copy)
- assert.NilError(t, err, "ReadFile")
- assert.DeepEqual(t, origBytes, copyBytes)
- origStat, err := os.Lstat(orig)
- assert.NilError(t, err, "Lstat")
- copyStat, err := os.Lstat(copy)
- assert.NilError(t, err, "Lstat")
- assert.Equal(t, origStat.Mode(), copyStat.Mode())
-}
-
-func assertDirMatches(t *testing.T, orig string, copy string) {
- t.Helper()
- origStat, err := os.Lstat(orig)
- assert.NilError(t, err, "Lstat")
- copyStat, err := os.Lstat(copy)
- assert.NilError(t, err, "Lstat")
- assert.Equal(t, origStat.Mode(), copyStat.Mode())
-}
diff --git a/cli/internal/fs/fs.go b/cli/internal/fs/fs.go
deleted file mode 100644
index 77804c0..0000000
--- a/cli/internal/fs/fs.go
+++ /dev/null
@@ -1,191 +0,0 @@
-package fs
-
-import (
- "io"
- "io/ioutil"
- "log"
- "os"
- "path/filepath"
- "runtime"
- "strings"
-
- "github.com/pkg/errors"
- "github.com/vercel/turbo/cli/internal/util"
-)
-
-// https://github.com/thought-machine/please/blob/master/src/fs/fs.go
-
-// DirPermissions are the default permission bits we apply to directories.
-const DirPermissions = os.ModeDir | 0775
-
-// EnsureDir ensures that the directory of the given file has been created.
-func EnsureDir(filename string) error {
- dir := filepath.Dir(filename)
- err := os.MkdirAll(dir, DirPermissions)
- if err != nil && FileExists(dir) {
- // It looks like this is a file and not a directory. Attempt to remove it; this can
- // happen in some cases if you change a rule from outputting a file to a directory.
- log.Printf("Attempting to remove file %s; a subdirectory is required", dir)
- if err2 := os.Remove(dir); err2 == nil {
- err = os.MkdirAll(dir, DirPermissions)
- } else {
- return err
- }
- }
- return err
-}
-
-var nonRelativeSentinel string = ".." + string(filepath.Separator)
-
-// DirContainsPath returns true if the path 'target' is contained within 'dir'
-// Expects both paths to be absolute and does not verify that either path exists.
-func DirContainsPath(dir string, target string) (bool, error) {
- // On windows, trying to get a relative path between files on different volumes
- // is an error. We don't care about the error, it's good enough for us to say
- // that one path doesn't contain the other if they're on different volumes.
- if runtime.GOOS == "windows" && filepath.VolumeName(dir) != filepath.VolumeName(target) {
- return false, nil
- }
- // In Go, filepath.Rel can return a path that starts with "../" or equivalent.
- // Checking filesystem-level contains can get extremely complicated
- // (see https://github.com/golang/dep/blob/f13583b555deaa6742f141a9c1185af947720d60/internal/fs/fs.go#L33)
- // As a compromise, rely on the stdlib to generate a relative path and then check
- // if the first step is "../".
- rel, err := filepath.Rel(dir, target)
- if err != nil {
- return false, err
- }
- return !strings.HasPrefix(rel, nonRelativeSentinel), nil
-}
-
-// PathExists returns true if the given path exists, as a file or a directory.
-func PathExists(filename string) bool {
- _, err := os.Lstat(filename)
- return err == nil
-}
-
-// FileExists returns true if the given path exists and is a file.
-func FileExists(filename string) bool {
- info, err := os.Lstat(filename)
- return err == nil && !info.IsDir()
-}
-
-// CopyFile copies a file from 'from' to 'to', with an attempt to perform a copy & rename
-// to avoid chaos if anything goes wrong partway.
-func CopyFile(from *LstatCachedFile, to string) error {
- fromMode, err := from.GetMode()
- if err != nil {
- return errors.Wrapf(err, "getting mode for %v", from.Path)
- }
- if fromMode&os.ModeSymlink != 0 {
- target, err := from.Path.Readlink()
- if err != nil {
- return errors.Wrapf(err, "reading link target for %v", from.Path)
- }
- if err := EnsureDir(to); err != nil {
- return err
- }
- if _, err := os.Lstat(to); err == nil {
- // target link file exist, should remove it first
- err := os.Remove(to)
- if err != nil {
- return err
- }
- }
- return os.Symlink(target, to)
- }
- fromFile, err := from.Path.Open()
- if err != nil {
- return err
- }
- defer util.CloseAndIgnoreError(fromFile)
- return writeFileFromStream(fromFile, to, fromMode)
-}
-
-// writeFileFromStream writes data from a reader to the file named 'to', with an attempt to perform
-// a copy & rename to avoid chaos if anything goes wrong partway.
-func writeFileFromStream(fromFile io.Reader, to string, mode os.FileMode) error {
- dir, file := filepath.Split(to)
- if dir != "" {
- if err := os.MkdirAll(dir, DirPermissions); err != nil {
- return err
- }
- }
- tempFile, err := ioutil.TempFile(dir, file)
- if err != nil {
- return err
- }
- if _, err := io.Copy(tempFile, fromFile); err != nil {
- return err
- }
- if err := tempFile.Close(); err != nil {
- return err
- }
- // OK, now file is written; adjust permissions appropriately.
- if mode == 0 {
- mode = 0664
- }
- if err := os.Chmod(tempFile.Name(), mode); err != nil {
- return err
- }
- // And move it to its final destination.
- return renameFile(tempFile.Name(), to)
-}
-
-// IsDirectory checks if a given path is a directory
-func IsDirectory(path string) bool {
- info, err := os.Stat(path)
- return err == nil && info.IsDir()
-}
-
-// Try to gracefully rename the file as the os.Rename does not work across
-// filesystems and on most Linux systems /tmp is mounted as tmpfs
-func renameFile(from, to string) (err error) {
- err = os.Rename(from, to)
- if err == nil {
- return nil
- }
- err = copyFile(from, to)
- if err != nil {
- return err
- }
- err = os.RemoveAll(from)
- if err != nil {
- return err
- }
- return nil
-}
-
-func copyFile(from, to string) (err error) {
- in, err := os.Open(from)
- if err != nil {
- return err
- }
- defer in.Close()
-
- out, err := os.Create(to)
- if err != nil {
- return err
- }
- defer func() {
- if e := out.Close(); e != nil {
- err = e
- }
- }()
-
- _, err = io.Copy(out, in)
- if err != nil {
- return err
- }
-
- si, err := os.Stat(from)
- if err != nil {
- return err
- }
- err = os.Chmod(to, si.Mode())
- if err != nil {
- return err
- }
-
- return nil
-}
diff --git a/cli/internal/fs/fs_test.go b/cli/internal/fs/fs_test.go
deleted file mode 100644
index 0598d43..0000000
--- a/cli/internal/fs/fs_test.go
+++ /dev/null
@@ -1,60 +0,0 @@
-package fs
-
-import (
- "path/filepath"
- "testing"
-)
-
-func Test_DirContainsPath(t *testing.T) {
- parent, err := filepath.Abs(filepath.Join("some", "path"))
- if err != nil {
- t.Fatalf("failed to construct parent path %v", err)
- }
- testcases := []struct {
- target []string
- want bool
- }{
- {
- []string{"..", "elsewhere"},
- false,
- },
- {
- []string{"sibling"},
- false,
- },
- {
- // The same path as parent
- []string{"some", "path"},
- true,
- },
- {
- []string{"some", "path", "..", "path", "inside", "parent"},
- true,
- },
- {
- []string{"some", "path", "inside", "..", "inside", "parent"},
- true,
- },
- {
- []string{"some", "path", "inside", "..", "..", "outside", "parent"},
- false,
- },
- {
- []string{"some", "pathprefix"},
- false,
- },
- }
- for _, tc := range testcases {
- target, err := filepath.Abs(filepath.Join(tc.target...))
- if err != nil {
- t.Fatalf("failed to construct path for %v: %v", tc.target, err)
- }
- got, err := DirContainsPath(parent, target)
- if err != nil {
- t.Fatalf("failed to check ")
- }
- if got != tc.want {
- t.Errorf("DirContainsPath(%v, %v) got %v, want %v", parent, target, got, tc.want)
- }
- }
-}
diff --git a/cli/internal/fs/fs_windows_test.go b/cli/internal/fs/fs_windows_test.go
deleted file mode 100644
index 4e71e2c..0000000
--- a/cli/internal/fs/fs_windows_test.go
+++ /dev/null
@@ -1,18 +0,0 @@
-//go:build windows
-// +build windows
-
-package fs
-
-import "testing"
-
-func TestDifferentVolumes(t *testing.T) {
- p1 := "C:\\some\\path"
- p2 := "D:\\other\\path"
- contains, err := DirContainsPath(p1, p2)
- if err != nil {
- t.Errorf("DirContainsPath got error %v, want <nil>", err)
- }
- if contains {
- t.Errorf("DirContainsPath got true, want false")
- }
-}
diff --git a/cli/internal/fs/get_turbo_data_dir_go.go b/cli/internal/fs/get_turbo_data_dir_go.go
deleted file mode 100644
index 2cf459a..0000000
--- a/cli/internal/fs/get_turbo_data_dir_go.go
+++ /dev/null
@@ -1,16 +0,0 @@
-//go:build go || !rust
-// +build go !rust
-
-package fs
-
-import (
- "github.com/adrg/xdg"
- "github.com/vercel/turbo/cli/internal/turbopath"
-)
-
-// GetTurboDataDir returns a directory outside of the repo
-// where turbo can store data files related to turbo.
-func GetTurboDataDir() turbopath.AbsoluteSystemPath {
- dataHome := AbsoluteSystemPathFromUpstream(xdg.DataHome)
- return dataHome.UntypedJoin("turborepo")
-}
diff --git a/cli/internal/fs/get_turbo_data_dir_rust.go b/cli/internal/fs/get_turbo_data_dir_rust.go
deleted file mode 100644
index dbc80f3..0000000
--- a/cli/internal/fs/get_turbo_data_dir_rust.go
+++ /dev/null
@@ -1,16 +0,0 @@
-//go:build rust
-// +build rust
-
-package fs
-
-import (
- "github.com/vercel/turbo/cli/internal/ffi"
- "github.com/vercel/turbo/cli/internal/turbopath"
-)
-
-// GetTurboDataDir returns a directory outside of the repo
-// where turbo can store data files related to turbo.
-func GetTurboDataDir() turbopath.AbsoluteSystemPath {
- dir := ffi.GetTurboDataDir()
- return turbopath.AbsoluteSystemPathFromUpstream(dir)
-}
diff --git a/cli/internal/fs/hash.go b/cli/internal/fs/hash.go
deleted file mode 100644
index fed7d87..0000000
--- a/cli/internal/fs/hash.go
+++ /dev/null
@@ -1,61 +0,0 @@
-package fs
-
-import (
- "crypto/sha1"
- "encoding/hex"
- "fmt"
- "io"
- "os"
- "strconv"
-
- "github.com/vercel/turbo/cli/internal/xxhash"
-)
-
-func HashObject(i interface{}) (string, error) {
- hash := xxhash.New()
-
- _, err := hash.Write([]byte(fmt.Sprintf("%v", i)))
-
- return hex.EncodeToString(hash.Sum(nil)), err
-}
-
-func HashFile(filePath string) (string, error) {
- file, err := os.Open(filePath)
- if err != nil {
- return "", err
- }
- defer file.Close()
-
- hash := xxhash.New()
- if _, err := io.Copy(hash, file); err != nil {
- return "", err
- }
-
- return hex.EncodeToString(hash.Sum(nil)), nil
-}
-
-// GitLikeHashFile is a function that mimics how Git
-// calculates the SHA1 for a file (or, in Git terms, a "blob") (without git)
-func GitLikeHashFile(filePath string) (string, error) {
- file, err := os.Open(filePath)
- if err != nil {
- return "", err
- }
- defer file.Close()
-
- stat, err := file.Stat()
- if err != nil {
- return "", err
- }
- hash := sha1.New()
- hash.Write([]byte("blob"))
- hash.Write([]byte(" "))
- hash.Write([]byte(strconv.FormatInt(stat.Size(), 10)))
- hash.Write([]byte{0})
-
- if _, err := io.Copy(hash, file); err != nil {
- return "", err
- }
-
- return hex.EncodeToString(hash.Sum(nil)), nil
-}
diff --git a/cli/internal/fs/hash_test.go b/cli/internal/fs/hash_test.go
deleted file mode 100644
index dd2fa84..0000000
--- a/cli/internal/fs/hash_test.go
+++ /dev/null
@@ -1,53 +0,0 @@
-package fs
-
-import (
- "testing"
-
- "gotest.tools/v3/assert"
-)
-
-const _numOfRuns = 20
-
-func Test_HashObjectStability(t *testing.T) {
- type TestCase struct {
- name string
- obj interface{}
- }
- type complexStruct struct {
- nested TaskOutputs
- foo string
- bar []string
- }
-
- testCases := []TestCase{
- {
- name: "task object",
- obj: TaskOutputs{
- Inclusions: []string{"foo", "bar"},
- Exclusions: []string{"baz"},
- },
- },
- {
- name: "complex struct",
- obj: complexStruct{
- nested: TaskOutputs{
- Exclusions: []string{"bar", "baz"},
- Inclusions: []string{"foo"},
- },
- foo: "a",
- bar: []string{"b", "c"},
- },
- },
- }
-
- for _, tc := range testCases {
- expectedHash, err := HashObject(tc.obj)
- assert.NilError(t, err, tc.name)
-
- for n := 0; n < _numOfRuns; n++ {
- hash, err := HashObject(tc.obj)
- assert.NilError(t, err, tc.name)
- assert.Equal(t, expectedHash, hash, tc.name)
- }
- }
-}
diff --git a/cli/internal/fs/lstat.go b/cli/internal/fs/lstat.go
deleted file mode 100644
index eff0810..0000000
--- a/cli/internal/fs/lstat.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package fs
-
-import (
- "io/fs"
- "os"
-
- "github.com/vercel/turbo/cli/internal/turbopath"
-)
-
-// LstatCachedFile maintains a cache of file info, mode and type for the given Path
-type LstatCachedFile struct {
- Path turbopath.AbsoluteSystemPath
- fileInfo fs.FileInfo
- fileMode *fs.FileMode
- fileType *fs.FileMode
-}
-
-// GetInfo returns, and caches the file info for the LstatCachedFile.Path
-func (file *LstatCachedFile) GetInfo() (fs.FileInfo, error) {
- if file.fileInfo != nil {
- return file.fileInfo, nil
- }
-
- err := file.lstat()
- if err != nil {
- return nil, err
- }
-
- return file.fileInfo, nil
-}
-
-// GetMode returns, and caches the file mode for the LstatCachedFile.Path
-func (file *LstatCachedFile) GetMode() (fs.FileMode, error) {
- if file.fileMode != nil {
- return *file.fileMode, nil
- }
-
- err := file.lstat()
- if err != nil {
- return 0, err
- }
-
- return *file.fileMode, nil
-}
-
-// GetType returns, and caches the type bits of (FileMode & os.ModeType) for the LstatCachedFile.Path
-func (file *LstatCachedFile) GetType() (fs.FileMode, error) {
- if file.fileType != nil {
- return *file.fileType, nil
- }
-
- err := file.lstat()
- if err != nil {
- return 0, err
- }
-
- return *file.fileType, nil
-}
-
-func (file *LstatCachedFile) lstat() error {
- fileInfo, err := file.Path.Lstat()
- if err != nil {
- return err
- }
-
- fileMode := fileInfo.Mode()
- fileModeType := fileMode & os.ModeType
-
- file.fileInfo = fileInfo
- file.fileMode = &fileMode
- file.fileType = &fileModeType
-
- return nil
-}
diff --git a/cli/internal/fs/package_json.go b/cli/internal/fs/package_json.go
deleted file mode 100644
index 883f7a4..0000000
--- a/cli/internal/fs/package_json.go
+++ /dev/null
@@ -1,142 +0,0 @@
-package fs
-
-import (
- "bytes"
- "encoding/json"
- "sync"
-
- "github.com/vercel/turbo/cli/internal/lockfile"
- "github.com/vercel/turbo/cli/internal/turbopath"
-)
-
-// PackageJSON represents NodeJS package.json
-type PackageJSON struct {
- Name string `json:"name"`
- Version string `json:"version"`
- Scripts map[string]string `json:"scripts"`
- Dependencies map[string]string `json:"dependencies"`
- DevDependencies map[string]string `json:"devDependencies"`
- OptionalDependencies map[string]string `json:"optionalDependencies"`
- PeerDependencies map[string]string `json:"peerDependencies"`
- PackageManager string `json:"packageManager"`
- Os []string `json:"os"`
- Workspaces Workspaces `json:"workspaces"`
- Private bool `json:"private"`
- // Exact JSON object stored in package.json including unknown fields
- // During marshalling struct fields will take priority over raw fields
- RawJSON map[string]interface{} `json:"-"`
-
- // relative path from repo root to the package.json file
- PackageJSONPath turbopath.AnchoredSystemPath `json:"-"`
- // relative path from repo root to the package
- Dir turbopath.AnchoredSystemPath `json:"-"`
- InternalDeps []string `json:"-"`
- UnresolvedExternalDeps map[string]string `json:"-"`
- TransitiveDeps []lockfile.Package `json:"-"`
- LegacyTurboConfig *TurboJSON `json:"turbo"`
- Mu sync.Mutex `json:"-"`
- ExternalDepsHash string `json:"-"`
-}
-
-type Workspaces []string
-
-type WorkspacesAlt struct {
- Packages []string `json:"packages,omitempty"`
-}
-
-func (r *Workspaces) UnmarshalJSON(data []byte) error {
- var tmp = &WorkspacesAlt{}
- if err := json.Unmarshal(data, tmp); err == nil {
- *r = Workspaces(tmp.Packages)
- return nil
- }
- var tempstr = []string{}
- if err := json.Unmarshal(data, &tempstr); err != nil {
- return err
- }
- *r = tempstr
- return nil
-}
-
-// ReadPackageJSON returns a struct of package.json
-func ReadPackageJSON(path turbopath.AbsoluteSystemPath) (*PackageJSON, error) {
- b, err := path.ReadFile()
- if err != nil {
- return nil, err
- }
- return UnmarshalPackageJSON(b)
-}
-
-// UnmarshalPackageJSON decodes a byte slice into a PackageJSON struct
-func UnmarshalPackageJSON(data []byte) (*PackageJSON, error) {
- var rawJSON map[string]interface{}
- if err := json.Unmarshal(data, &rawJSON); err != nil {
- return nil, err
- }
-
- pkgJSON := &PackageJSON{}
- if err := json.Unmarshal(data, &pkgJSON); err != nil {
- return nil, err
- }
- pkgJSON.RawJSON = rawJSON
-
- return pkgJSON, nil
-}
-
-// MarshalPackageJSON Serialize PackageJSON to a slice of bytes
-func MarshalPackageJSON(pkgJSON *PackageJSON) ([]byte, error) {
- structuredContent, err := json.Marshal(pkgJSON)
- if err != nil {
- return nil, err
- }
- var structuredFields map[string]interface{}
- if err := json.Unmarshal(structuredContent, &structuredFields); err != nil {
- return nil, err
- }
-
- fieldsToSerialize := make(map[string]interface{}, len(pkgJSON.RawJSON))
-
- // copy pkgJSON.RawJSON
- for key, value := range pkgJSON.RawJSON {
- fieldsToSerialize[key] = value
- }
-
- for key, value := range structuredFields {
- if isEmpty(value) {
- delete(fieldsToSerialize, key)
- } else {
- fieldsToSerialize[key] = value
- }
- }
-
- var b bytes.Buffer
- encoder := json.NewEncoder(&b)
- encoder.SetEscapeHTML(false)
- encoder.SetIndent("", " ")
- if err := encoder.Encode(fieldsToSerialize); err != nil {
- return nil, err
- }
-
- return b.Bytes(), nil
-}
-
-func isEmpty(value interface{}) bool {
- if value == nil {
- return true
- }
- switch s := value.(type) {
- case string:
- return s == ""
- case bool:
- return !s
- case []string:
- return len(s) == 0
- case map[string]interface{}:
- return len(s) == 0
- case Workspaces:
- return len(s) == 0
- default:
- // Assume any unknown types aren't empty
- return false
- }
-}
diff --git a/cli/internal/fs/package_json_test.go b/cli/internal/fs/package_json_test.go
deleted file mode 100644
index 3c16620..0000000
--- a/cli/internal/fs/package_json_test.go
+++ /dev/null
@@ -1,174 +0,0 @@
-package fs
-
-import (
- "testing"
-
- "gotest.tools/v3/assert"
-)
-
-func Test_UnmarshalPackageJSON(t *testing.T) {
- type Case struct {
- name string
- json string
- expectedFields *PackageJSON
- }
-
- testCases := []Case{
- {
- name: "basic types are in raw and processed",
- json: `{"name":"foo","version":"1.2.3"}`,
- expectedFields: &PackageJSON{
- Name: "foo",
- Version: "1.2.3",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "version": "1.2.3",
- },
- },
- },
- {
- name: "map types get copied",
- json: `{"dependencies":{"foo":"1.2.3"},"devDependencies":{"bar": "^1.0.0"}}`,
- expectedFields: &PackageJSON{
- Dependencies: map[string]string{"foo": "1.2.3"},
- DevDependencies: map[string]string{"bar": "^1.0.0"},
- RawJSON: map[string]interface{}{
- "dependencies": map[string]interface{}{"foo": "1.2.3"},
- "devDependencies": map[string]interface{}{"bar": "^1.0.0"},
- },
- },
- },
- {
- name: "array types get copied",
- json: `{"os":["linux", "windows"]}`,
- expectedFields: &PackageJSON{
- Os: []string{"linux", "windows"},
- RawJSON: map[string]interface{}{
- "os": []interface{}{"linux", "windows"},
- },
- },
- },
- }
-
- for _, testCase := range testCases {
- actual, err := UnmarshalPackageJSON([]byte(testCase.json))
- assert.NilError(t, err, testCase.name)
- assertPackageJSONEqual(t, actual, testCase.expectedFields)
- }
-}
-
-func Test_MarshalPackageJSON(t *testing.T) {
- type TestCase struct {
- name string
- input *PackageJSON
- expected *PackageJSON
- }
-
- testCases := []TestCase{
- {
- name: "roundtrip should have no effect",
- input: &PackageJSON{
- Name: "foo",
- Version: "1.2.3",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "version": "1.2.3",
- },
- },
- expected: &PackageJSON{
- Name: "foo",
- Version: "1.2.3",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "version": "1.2.3",
- },
- },
- },
- {
- name: "structured fields should take priority over raw values",
- input: &PackageJSON{
- Name: "foo",
- Version: "2.3.4",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "version": "1.2.3",
- },
- },
- expected: &PackageJSON{
- Name: "foo",
- Version: "2.3.4",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "version": "2.3.4",
- },
- },
- },
- {
- name: "empty structured fields don't get serialized",
- input: &PackageJSON{
- Name: "foo",
- Version: "",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "version": "1.2.3",
- },
- },
- expected: &PackageJSON{
- Name: "foo",
- Version: "",
- RawJSON: map[string]interface{}{
- "name": "foo",
- },
- },
- },
- {
- name: "unstructured fields survive the round trip",
- input: &PackageJSON{
- Name: "foo",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "special-field": "special-value",
- "special-config": map[string]interface{}{
- "flag": true,
- "value": "toggled",
- },
- },
- },
- expected: &PackageJSON{
- Name: "foo",
- RawJSON: map[string]interface{}{
- "name": "foo",
- "special-field": "special-value",
- "special-config": map[string]interface{}{
- "flag": true,
- "value": "toggled",
- },
- },
- },
- },
- }
-
- for _, testCase := range testCases {
- serializedInput, err := MarshalPackageJSON(testCase.input)
- assert.NilError(t, err, testCase.name)
- actual, err := UnmarshalPackageJSON(serializedInput)
- assert.NilError(t, err, testCase.name)
- assertPackageJSONEqual(t, actual, testCase.expected)
- }
-}
-
-// Asserts that the data section of two PackageJSON structs are equal
-func assertPackageJSONEqual(t *testing.T, x *PackageJSON, y *PackageJSON) {
- t.Helper()
- assert.Equal(t, x.Name, y.Name)
- assert.Equal(t, x.Version, y.Version)
- assert.DeepEqual(t, x.Scripts, y.Scripts)
- assert.DeepEqual(t, x.Dependencies, y.Dependencies)
- assert.DeepEqual(t, x.DevDependencies, y.DevDependencies)
- assert.DeepEqual(t, x.OptionalDependencies, y.OptionalDependencies)
- assert.DeepEqual(t, x.PeerDependencies, y.PeerDependencies)
- assert.Equal(t, x.PackageManager, y.PackageManager)
- assert.DeepEqual(t, x.Workspaces, y.Workspaces)
- assert.DeepEqual(t, x.Private, y.Private)
- assert.DeepEqual(t, x.RawJSON, y.RawJSON)
-}
diff --git a/cli/internal/fs/path.go b/cli/internal/fs/path.go
deleted file mode 100644
index 2023d69..0000000
--- a/cli/internal/fs/path.go
+++ /dev/null
@@ -1,113 +0,0 @@
-package fs
-
-import (
- "fmt"
- iofs "io/fs"
- "os"
- "path/filepath"
- "reflect"
-
- "github.com/adrg/xdg"
- "github.com/vercel/turbo/cli/internal/turbopath"
-)
-
-// CheckedToAbsoluteSystemPath inspects a string and determines if it is an absolute path.
-func CheckedToAbsoluteSystemPath(s string) (turbopath.AbsoluteSystemPath, error) {
- if filepath.IsAbs(s) {
- return turbopath.AbsoluteSystemPath(s), nil
- }
- return "", fmt.Errorf("%v is not an absolute path", s)
-}
-
-// ResolveUnknownPath returns unknown if it is an absolute path, otherwise, it
-// assumes unknown is a path relative to the given root.
-func ResolveUnknownPath(root turbopath.AbsoluteSystemPath, unknown string) turbopath.AbsoluteSystemPath {
- if filepath.IsAbs(unknown) {
- return turbopath.AbsoluteSystemPath(unknown)
- }
- return root.UntypedJoin(unknown)
-}
-
-// UnsafeToAbsoluteSystemPath directly converts a string to an AbsoluteSystemPath
-func UnsafeToAbsoluteSystemPath(s string) turbopath.AbsoluteSystemPath {
- return turbopath.AbsoluteSystemPath(s)
-}
-
-// UnsafeToAnchoredSystemPath directly converts a string to an AbsoluteSystemPath
-func UnsafeToAnchoredSystemPath(s string) turbopath.AnchoredSystemPath {
- return turbopath.AnchoredSystemPath(s)
-}
-
-// AbsoluteSystemPathFromUpstream is used to mark return values from APIs that we
-// expect to give us absolute paths. No checking is performed.
-// Prefer to use this over a cast to maintain the search-ability of interfaces
-// into and out of the turbopath.AbsoluteSystemPath type.
-func AbsoluteSystemPathFromUpstream(s string) turbopath.AbsoluteSystemPath {
- return turbopath.AbsoluteSystemPath(s)
-}
-
-// GetCwd returns the calculated working directory after traversing symlinks.
-func GetCwd(cwdRaw string) (turbopath.AbsoluteSystemPath, error) {
- if cwdRaw == "" {
- var err error
- cwdRaw, err = os.Getwd()
- if err != nil {
- return "", err
- }
- }
- // We evaluate symlinks here because the package managers
- // we support do the same.
- cwdRaw, err := filepath.EvalSymlinks(cwdRaw)
- if err != nil {
- return "", fmt.Errorf("evaluating symlinks in cwd: %w", err)
- }
- cwd, err := CheckedToAbsoluteSystemPath(cwdRaw)
- if err != nil {
- return "", fmt.Errorf("cwd is not an absolute path %v: %v", cwdRaw, err)
- }
- return cwd, nil
-}
-
-// GetVolumeRoot returns the root directory given an absolute path.
-func GetVolumeRoot(absolutePath string) string {
- return filepath.VolumeName(absolutePath) + string(os.PathSeparator)
-}
-
-// CreateDirFSAtRoot creates an `os.dirFS` instance at the root of the
-// volume containing the specified path.
-func CreateDirFSAtRoot(absolutePath string) iofs.FS {
- return os.DirFS(GetVolumeRoot(absolutePath))
-}
-
-// GetDirFSRootPath returns the root path of a os.dirFS.
-func GetDirFSRootPath(fsys iofs.FS) string {
- // We can't typecheck fsys to enforce using an `os.dirFS` because the
- // type isn't exported from `os`. So instead, reflection. 🤷‍♂️
-
- fsysType := reflect.TypeOf(fsys).Name()
- if fsysType != "dirFS" {
- // This is not a user error, fail fast
- panic("GetDirFSRootPath must receive an os.dirFS")
- }
-
- // The underlying type is a string; this is the original path passed in.
- return reflect.ValueOf(fsys).String()
-}
-
-// IofsRelativePath calculates a `os.dirFS`-friendly path from an absolute system path.
-func IofsRelativePath(fsysRoot string, absolutePath string) (string, error) {
- return filepath.Rel(fsysRoot, absolutePath)
-}
-
-// TempDir returns the absolute path of a directory with the given name
-// under the system's default temp directory location
-func TempDir(subDir string) turbopath.AbsoluteSystemPath {
- return turbopath.AbsoluteSystemPath(os.TempDir()).UntypedJoin(subDir)
-}
-
-// GetUserConfigDir returns the platform-specific common location
-// for configuration files that belong to a user.
-func GetUserConfigDir() turbopath.AbsoluteSystemPath {
- configHome := AbsoluteSystemPathFromUpstream(xdg.ConfigHome)
- return configHome.UntypedJoin("turborepo")
-}
diff --git a/cli/internal/fs/testdata/both/package.json b/cli/internal/fs/testdata/both/package.json
deleted file mode 100644
index 03534b7..0000000
--- a/cli/internal/fs/testdata/both/package.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "turbo": {
- "pipeline": {
- "build": {}
- }
- }
-}
diff --git a/cli/internal/fs/testdata/both/turbo.json b/cli/internal/fs/testdata/both/turbo.json
deleted file mode 100644
index 721e897..0000000
--- a/cli/internal/fs/testdata/both/turbo.json
+++ /dev/null
@@ -1,18 +0,0 @@
-// mocked test comment
-{
- "pipeline": {
- "build": {
- // mocked test comment
- "dependsOn": [
- // mocked test comment
- "^build"
- ],
- "outputs": ["dist/**", ".next/**", "!dist/assets/**"],
- "outputMode": "new-only"
- } // mocked test comment
- },
- "remoteCache": {
- "teamId": "team_id",
- "signature": true
- }
-}
diff --git a/cli/internal/fs/testdata/correct/turbo.json b/cli/internal/fs/testdata/correct/turbo.json
deleted file mode 100644
index e22cde2..0000000
--- a/cli/internal/fs/testdata/correct/turbo.json
+++ /dev/null
@@ -1,49 +0,0 @@
-// mocked test comment
-{
- "pipeline": {
- "build": {
- "experimentalPassthroughEnv": ["GITHUB_TOKEN"],
- // mocked test comment
- "dependsOn": [
- // mocked test comment
- "^build"
- ],
- "outputs": ["dist/**", "!dist/assets/**", ".next/**"],
- "outputMode": "new-only"
- }, // mocked test comment
- "lint": {
- "outputs": [],
- "dependsOn": ["$MY_VAR"],
- "cache": true,
- "outputMode": "new-only"
- },
- "dev": {
- "cache": false,
- "outputMode": "full"
- },
- /* mocked test comment */
- "publish": {
- "outputs": ["dist/**"],
- "inputs": [
- /*
- mocked test comment
- */
- "build/**/*"
- ],
- "dependsOn": [
- /* mocked test comment */ "^publish",
- "^build",
- "build",
- "admin#lint"
- ],
- "cache": false
- }
- },
- "globalDependencies": ["some-file", "../another-dir/**", "$GLOBAL_ENV_VAR"],
- "globlaEnv": ["SOME_VAR", "ANOTHER_VAR"],
- "experimentalGlobalPassThroughEnv": ["AWS_SECRET_KEY"],
- "remoteCache": {
- "teamId": "team_id",
- "signature": true
- }
-}
diff --git a/cli/internal/fs/testdata/invalid-env-1/turbo.json b/cli/internal/fs/testdata/invalid-env-1/turbo.json
deleted file mode 100644
index e4a6517..0000000
--- a/cli/internal/fs/testdata/invalid-env-1/turbo.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "pipeline": {
- "task1": {
- // all invalid value
- "env": ["$A", "$B"]
- }
- }
-}
diff --git a/cli/internal/fs/testdata/invalid-env-2/turbo.json b/cli/internal/fs/testdata/invalid-env-2/turbo.json
deleted file mode 100644
index 92eec96..0000000
--- a/cli/internal/fs/testdata/invalid-env-2/turbo.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
- "pipeline": {
- "task1": {
- // Mixed values
- "env": ["$A", "B"]
- }
- }
-}
diff --git a/cli/internal/fs/testdata/invalid-global-env/turbo.json b/cli/internal/fs/testdata/invalid-global-env/turbo.json
deleted file mode 100644
index 2ae9ff9..0000000
--- a/cli/internal/fs/testdata/invalid-global-env/turbo.json
+++ /dev/null
@@ -1,11 +0,0 @@
-{
- // Both global declarations with duplicates
- "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"],
- // some invalid values
- "globalEnv": ["FOO", "BAZ", "$QUX"],
- "pipeline": {
- "task1": {
- "dependsOn": ["$A"]
- }
- }
-}
diff --git a/cli/internal/fs/testdata/legacy-env/turbo.json b/cli/internal/fs/testdata/legacy-env/turbo.json
deleted file mode 100644
index 6b082c4..0000000
--- a/cli/internal/fs/testdata/legacy-env/turbo.json
+++ /dev/null
@@ -1,34 +0,0 @@
-// mocked test comment
-{
- // Both global declarations with duplicates and with
- "globalDependencies": ["$FOO", "$BAR", "somefile.txt", "somefile.txt"],
- "globalEnv": ["FOO", "BAZ", "QUX"],
- "pipeline": {
- // Only legacy declaration
- "task1": {
- "dependsOn": ["$A"]
- },
- // Only new declaration
- "task2": {
- "env": ["A"]
- },
- // Same var declared in both
- "task3": {
- "dependsOn": ["$A"],
- "env": ["A"]
- },
- // Different vars declared in both
- "task4": {
- "dependsOn": ["$A"],
- "env": ["B"]
- },
-
- // some edge cases
- "task6": { "env": ["A", "B", "C"], "dependsOn": ["$D", "$E", "$F"] },
- "task7": { "env": ["A", "B", "C"], "dependsOn": ["$A", "$B", "$C"] },
- "task8": { "env": ["A", "B", "C"], "dependsOn": ["A", "B", "C"] },
- "task9": { "env": [], "dependsOn": ["$A"] },
- "task10": { "env": ["A", "A"], "dependsOn": ["$A", "$A"] },
- "task11": { "env": ["A", "A"], "dependsOn": ["$B", "$B"] }
- }
-}
diff --git a/cli/internal/fs/testdata/legacy-only/package.json b/cli/internal/fs/testdata/legacy-only/package.json
deleted file mode 100644
index 03534b7..0000000
--- a/cli/internal/fs/testdata/legacy-only/package.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "turbo": {
- "pipeline": {
- "build": {}
- }
- }
-}
diff --git a/cli/internal/fs/turbo_json.go b/cli/internal/fs/turbo_json.go
deleted file mode 100644
index 71ef29d..0000000
--- a/cli/internal/fs/turbo_json.go
+++ /dev/null
@@ -1,741 +0,0 @@
-package fs
-
-import (
- "encoding/json"
- "fmt"
- "io/ioutil"
- "log"
- "os"
- "path/filepath"
- "sort"
- "strings"
-
- "github.com/muhammadmuzzammil1998/jsonc"
- "github.com/pkg/errors"
- "github.com/vercel/turbo/cli/internal/turbopath"
- "github.com/vercel/turbo/cli/internal/util"
-)
-
-const (
- configFile = "turbo.json"
- envPipelineDelimiter = "$"
- topologicalPipelineDelimiter = "^"
-)
-
-type rawTurboJSON struct {
- // Global root filesystem dependencies
- GlobalDependencies []string `json:"globalDependencies,omitempty"`
- // Global env
- GlobalEnv []string `json:"globalEnv,omitempty"`
-
- // Global passthrough env
- GlobalPassthroughEnv []string `json:"experimentalGlobalPassThroughEnv,omitempty"`
-
- // Pipeline is a map of Turbo pipeline entries which define the task graph
- // and cache behavior on a per task or per package-task basis.
- Pipeline Pipeline `json:"pipeline"`
- // Configuration options when interfacing with the remote cache
- RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"`
-
- // Extends can be the name of another workspace
- Extends []string `json:"extends,omitempty"`
-}
-
-// pristineTurboJSON is used when marshaling a TurboJSON object into a turbo.json string
-// Notably, it includes a PristinePipeline instead of the regular Pipeline. (i.e. TaskDefinition
-// instead of BookkeepingTaskDefinition.)
-type pristineTurboJSON struct {
- GlobalDependencies []string `json:"globalDependencies,omitempty"`
- GlobalEnv []string `json:"globalEnv,omitempty"`
- GlobalPassthroughEnv []string `json:"experimentalGlobalPassThroughEnv,omitempty"`
- Pipeline PristinePipeline `json:"pipeline"`
- RemoteCacheOptions RemoteCacheOptions `json:"remoteCache,omitempty"`
- Extends []string `json:"extends,omitempty"`
-}
-
-// TurboJSON represents a turbo.json configuration file
-type TurboJSON struct {
- GlobalDeps []string
- GlobalEnv []string
- GlobalPassthroughEnv []string
- Pipeline Pipeline
- RemoteCacheOptions RemoteCacheOptions
-
- // A list of Workspace names
- Extends []string
-}
-
-// RemoteCacheOptions is a struct for deserializing .remoteCache of configFile
-type RemoteCacheOptions struct {
- TeamID string `json:"teamId,omitempty"`
- Signature bool `json:"signature,omitempty"`
-}
-
-// rawTaskWithDefaults exists to Marshal (i.e. turn a TaskDefinition into json).
-// We use this for printing ResolvedTaskConfiguration, because we _want_ to show
-// the user the default values for key they have not configured.
-type rawTaskWithDefaults struct {
- Outputs []string `json:"outputs"`
- Cache *bool `json:"cache"`
- DependsOn []string `json:"dependsOn"`
- Inputs []string `json:"inputs"`
- OutputMode util.TaskOutputMode `json:"outputMode"`
- PassthroughEnv []string `json:"experimentalPassThroughEnv,omitempty"`
- Env []string `json:"env"`
- Persistent bool `json:"persistent"`
-}
-
-// rawTask exists to Unmarshal from json. When fields are omitted, we _want_
-// them to be missing, so that we can distinguish missing from empty value.
-type rawTask struct {
- Outputs []string `json:"outputs,omitempty"`
- Cache *bool `json:"cache,omitempty"`
- DependsOn []string `json:"dependsOn,omitempty"`
- Inputs []string `json:"inputs,omitempty"`
- OutputMode *util.TaskOutputMode `json:"outputMode,omitempty"`
- Env []string `json:"env,omitempty"`
- PassthroughEnv []string `json:"experimentalPassthroughEnv,omitempty"`
- Persistent *bool `json:"persistent,omitempty"`
-}
-
-// taskDefinitionHashable exists as a definition for PristinePipeline, which is used down
-// stream for calculating the global hash. We want to exclude experimental fields here
-// because we don't want experimental fields to be part of the global hash.
-type taskDefinitionHashable struct {
- Outputs TaskOutputs
- ShouldCache bool
- EnvVarDependencies []string
- TopologicalDependencies []string
- TaskDependencies []string
- Inputs []string
- OutputMode util.TaskOutputMode
- Persistent bool
-}
-
-// taskDefinitionExperiments is a list of config fields in a task definition that are considered
-// experimental. We keep these separated so we can compute a global hash without these.
-type taskDefinitionExperiments struct {
- PassthroughEnv []string
-}
-
-// PristinePipeline is a map of task names to TaskDefinition or taskDefinitionHashable.
-// Depending on whether any experimental fields are defined, we will use either struct.
-// The purpose is to omit experimental fields when making a pristine version, so that
-// it doesn't show up in --dry/--summarize output or affect the global hash.
-type PristinePipeline map[string]interface{}
-
-// Pipeline is a struct for deserializing .pipeline in configFile
-type Pipeline map[string]BookkeepingTaskDefinition
-
-// BookkeepingTaskDefinition holds the underlying TaskDefinition and some bookkeeping data
-// about the TaskDefinition. This wrapper struct allows us to leave TaskDefinition untouched.
-type BookkeepingTaskDefinition struct {
- definedFields util.Set
- experimentalFields util.Set
- experimental taskDefinitionExperiments
- TaskDefinition taskDefinitionHashable
-}
-
-// TaskDefinition is a representation of the configFile pipeline for further computation.
-type TaskDefinition struct {
- Outputs TaskOutputs
- ShouldCache bool
-
- // This field is custom-marshalled from rawTask.Env and rawTask.DependsOn
- EnvVarDependencies []string
-
- // rawTask.PassthroughEnv
- PassthroughEnv []string
-
- // TopologicalDependencies are tasks from package dependencies.
- // E.g. "build" is a topological dependency in:
- // dependsOn: ['^build'].
- // This field is custom-marshalled from rawTask.DependsOn
- TopologicalDependencies []string
-
- // TaskDependencies are anything that is not a topological dependency
- // E.g. both something and //whatever are TaskDependencies in:
- // dependsOn: ['something', '//whatever']
- // This field is custom-marshalled from rawTask.DependsOn
- TaskDependencies []string
-
- // Inputs indicate the list of files this Task depends on. If any of those files change
- // we can conclude that any cached outputs or logs for this Task should be invalidated.
- Inputs []string
-
- // OutputMode determins how we should log the output.
- OutputMode util.TaskOutputMode
-
- // Persistent indicates whether the Task is expected to exit or not
- // Tasks marked Persistent do not exit (e.g. --watch mode or dev servers)
- Persistent bool
-}
-
-// GetTask returns a TaskDefinition based on the ID (package#task format) or name (e.g. "build")
-func (pc Pipeline) GetTask(taskID string, taskName string) (*BookkeepingTaskDefinition, error) {
- // first check for package-tasks
- taskDefinition, ok := pc[taskID]
- if !ok {
- // then check for regular tasks
- fallbackTaskDefinition, notcool := pc[taskName]
- // if neither, then bail
- if !notcool {
- // Return an empty TaskDefinition
- return nil, fmt.Errorf("Could not find task \"%s\" in pipeline", taskID)
- }
-
- // override if we need to...
- taskDefinition = fallbackTaskDefinition
- }
-
- return &taskDefinition, nil
-}
-
-// LoadTurboConfig loads, or optionally, synthesizes a TurboJSON instance
-func LoadTurboConfig(dir turbopath.AbsoluteSystemPath, rootPackageJSON *PackageJSON, includeSynthesizedFromRootPackageJSON bool) (*TurboJSON, error) {
- // If the root package.json stil has a `turbo` key, log a warning and remove it.
- if rootPackageJSON.LegacyTurboConfig != nil {
- log.Printf("[WARNING] \"turbo\" in package.json is no longer supported. Migrate to %s by running \"npx @turbo/codemod create-turbo-config\"\n", configFile)
- rootPackageJSON.LegacyTurboConfig = nil
- }
-
- var turboJSON *TurboJSON
- turboFromFiles, err := readTurboConfig(dir.UntypedJoin(configFile))
-
- if !includeSynthesizedFromRootPackageJSON && err != nil {
- // If the file didn't exist, throw a custom error here instead of propagating
- if errors.Is(err, os.ErrNotExist) {
- return nil, errors.Wrap(err, fmt.Sprintf("Could not find %s. Follow directions at https://turbo.build/repo/docs to create one", configFile))
-
- }
-
- // There was an error, and we don't have any chance of recovering
- // because we aren't synthesizing anything
- return nil, err
- } else if !includeSynthesizedFromRootPackageJSON {
- // We're not synthesizing anything and there was no error, we're done
- return turboFromFiles, nil
- } else if errors.Is(err, os.ErrNotExist) {
- // turbo.json doesn't exist, but we're going try to synthesize something
- turboJSON = &TurboJSON{
- Pipeline: make(Pipeline),
- }
- } else if err != nil {
- // some other happened, we can't recover
- return nil, err
- } else {
- // we're synthesizing, but we have a starting point
- // Note: this will have to change to support task inference in a monorepo
- // for now, we're going to error on any "root" tasks and turn non-root tasks into root tasks
- pipeline := make(Pipeline)
- for taskID, taskDefinition := range turboFromFiles.Pipeline {
- if util.IsPackageTask(taskID) {
- return nil, fmt.Errorf("Package tasks (<package>#<task>) are not allowed in single-package repositories: found %v", taskID)
- }
- pipeline[util.RootTaskID(taskID)] = taskDefinition
- }
- turboJSON = turboFromFiles
- turboJSON.Pipeline = pipeline
- }
-
- for scriptName := range rootPackageJSON.Scripts {
- if !turboJSON.Pipeline.HasTask(scriptName) {
- taskName := util.RootTaskID(scriptName)
- // Explicitly set ShouldCache to false in this definition and add the bookkeeping fields
- // so downstream we can pretend that it was set on purpose (as if read from a config file)
- // rather than defaulting to the 0-value of a boolean field.
- turboJSON.Pipeline[taskName] = BookkeepingTaskDefinition{
- definedFields: util.SetFromStrings([]string{"ShouldCache"}),
- TaskDefinition: taskDefinitionHashable{
- ShouldCache: false,
- },
- }
- }
- }
- return turboJSON, nil
-}
-
-// TurboJSONValidation is the signature for a validation function passed to Validate()
-type TurboJSONValidation func(*TurboJSON) []error
-
-// Validate calls an array of validation functions on the TurboJSON struct.
-// The validations can be customized by the caller.
-func (tj *TurboJSON) Validate(validations []TurboJSONValidation) []error {
- allErrors := []error{}
- for _, validation := range validations {
- errors := validation(tj)
- allErrors = append(allErrors, errors...)
- }
-
- return allErrors
-}
-
-// TaskOutputs represents the patterns for including and excluding files from outputs
-type TaskOutputs struct {
- Inclusions []string
- Exclusions []string
-}
-
-// Sort contents of task outputs
-func (to TaskOutputs) Sort() TaskOutputs {
- var inclusions []string
- var exclusions []string
- copy(inclusions, to.Inclusions)
- copy(exclusions, to.Exclusions)
- sort.Strings(inclusions)
- sort.Strings(exclusions)
- return TaskOutputs{Inclusions: inclusions, Exclusions: exclusions}
-}
-
-// readTurboConfig reads turbo.json from a provided path
-func readTurboConfig(turboJSONPath turbopath.AbsoluteSystemPath) (*TurboJSON, error) {
- // If the configFile exists, use that
- if turboJSONPath.FileExists() {
- turboJSON, err := readTurboJSON(turboJSONPath)
- if err != nil {
- return nil, fmt.Errorf("%s: %w", configFile, err)
- }
-
- return turboJSON, nil
- }
-
- // If there's no turbo.json, return an error.
- return nil, os.ErrNotExist
-}
-
-// readTurboJSON reads the configFile in to a struct
-func readTurboJSON(path turbopath.AbsoluteSystemPath) (*TurboJSON, error) {
- file, err := path.Open()
- if err != nil {
- return nil, err
- }
- var turboJSON *TurboJSON
- data, err := ioutil.ReadAll(file)
- if err != nil {
- return nil, err
- }
-
- err = jsonc.Unmarshal(data, &turboJSON)
-
- if err != nil {
- return nil, err
- }
-
- return turboJSON, nil
-}
-
-// GetTaskDefinition returns a TaskDefinition from a serialized definition in configFile
-func (pc Pipeline) GetTaskDefinition(taskID string) (TaskDefinition, bool) {
- if entry, ok := pc[taskID]; ok {
- return entry.GetTaskDefinition(), true
- }
- _, task := util.GetPackageTaskFromId(taskID)
- entry, ok := pc[task]
- return entry.GetTaskDefinition(), ok
-}
-
-// HasTask returns true if the given task is defined in the pipeline, either directly or
-// via a package task (`pkg#task`)
-func (pc Pipeline) HasTask(task string) bool {
- for key := range pc {
- if key == task {
- return true
- }
- if util.IsPackageTask(key) {
- _, taskName := util.GetPackageTaskFromId(key)
- if taskName == task {
- return true
- }
- }
- }
- return false
-}
-
-// Pristine returns a PristinePipeline, this is used for printing to console and pruning
-func (pc Pipeline) Pristine() PristinePipeline {
- pristine := PristinePipeline{}
- for taskName, taskDef := range pc {
- // If there are any experimental fields, we will include them with 0-values
- // if there aren't, we will omit them entirely
- if taskDef.hasExperimentalFields() {
- pristine[taskName] = taskDef.GetTaskDefinition() // merges experimental fields in
- } else {
- pristine[taskName] = taskDef.TaskDefinition // has no experimental fields
- }
- }
- return pristine
-}
-
-// hasField checks the internal bookkeeping definedFields field to
-// see whether a field was actually in the underlying turbo.json
-// or whether it was initialized with its 0-value.
-func (btd BookkeepingTaskDefinition) hasField(fieldName string) bool {
- return btd.definedFields.Includes(fieldName) || btd.experimentalFields.Includes(fieldName)
-}
-
-// hasExperimentalFields keeps track of whether any experimental fields were found
-func (btd BookkeepingTaskDefinition) hasExperimentalFields() bool {
- return len(btd.experimentalFields) > 0
-}
-
-// GetTaskDefinition gets a TaskDefinition by merging the experimental and non-experimental fields
-// into a single representation to use downstream.
-func (btd BookkeepingTaskDefinition) GetTaskDefinition() TaskDefinition {
- return TaskDefinition{
- Outputs: btd.TaskDefinition.Outputs,
- ShouldCache: btd.TaskDefinition.ShouldCache,
- EnvVarDependencies: btd.TaskDefinition.EnvVarDependencies,
- TopologicalDependencies: btd.TaskDefinition.TopologicalDependencies,
- TaskDependencies: btd.TaskDefinition.TaskDependencies,
- Inputs: btd.TaskDefinition.Inputs,
- OutputMode: btd.TaskDefinition.OutputMode,
- Persistent: btd.TaskDefinition.Persistent,
- // From experimental fields
- PassthroughEnv: btd.experimental.PassthroughEnv,
- }
-}
-
-// MergeTaskDefinitions accepts an array of BookkeepingTaskDefinitions and merges them into
-// a single TaskDefinition. It uses the bookkeeping definedFields to determine which fields should
-// be overwritten and when 0-values should be respected.
-func MergeTaskDefinitions(taskDefinitions []BookkeepingTaskDefinition) (*TaskDefinition, error) {
- // Start with an empty definition
- mergedTaskDefinition := &TaskDefinition{}
-
- // Set the default, because the 0-value will be false, and if no turbo.jsons had
- // this field set for this task, we want it to be true.
- mergedTaskDefinition.ShouldCache = true
-
- // For each of the TaskDefinitions we know of, merge them in
- for _, bookkeepingTaskDef := range taskDefinitions {
- taskDef := bookkeepingTaskDef.GetTaskDefinition()
-
- if bookkeepingTaskDef.hasField("Outputs") {
- mergedTaskDefinition.Outputs = taskDef.Outputs
- }
-
- if bookkeepingTaskDef.hasField("ShouldCache") {
- mergedTaskDefinition.ShouldCache = taskDef.ShouldCache
- }
-
- if bookkeepingTaskDef.hasField("EnvVarDependencies") {
- mergedTaskDefinition.EnvVarDependencies = taskDef.EnvVarDependencies
- }
-
- if bookkeepingTaskDef.hasField("PassthroughEnv") {
- mergedTaskDefinition.PassthroughEnv = taskDef.PassthroughEnv
- }
-
- if bookkeepingTaskDef.hasField("DependsOn") {
- mergedTaskDefinition.TopologicalDependencies = taskDef.TopologicalDependencies
- }
-
- if bookkeepingTaskDef.hasField("DependsOn") {
- mergedTaskDefinition.TaskDependencies = taskDef.TaskDependencies
- }
-
- if bookkeepingTaskDef.hasField("Inputs") {
- mergedTaskDefinition.Inputs = taskDef.Inputs
- }
-
- if bookkeepingTaskDef.hasField("OutputMode") {
- mergedTaskDefinition.OutputMode = taskDef.OutputMode
- }
- if bookkeepingTaskDef.hasField("Persistent") {
- mergedTaskDefinition.Persistent = taskDef.Persistent
- }
- }
-
- return mergedTaskDefinition, nil
-}
-
-// UnmarshalJSON deserializes a single task definition from
-// turbo.json into a TaskDefinition struct
-func (btd *BookkeepingTaskDefinition) UnmarshalJSON(data []byte) error {
- task := rawTask{}
- if err := json.Unmarshal(data, &task); err != nil {
- return err
- }
-
- btd.definedFields = util.Set{}
- btd.experimentalFields = util.Set{}
-
- if task.Outputs != nil {
- var inclusions []string
- var exclusions []string
- // Assign a bookkeeping field so we know that there really were
- // outputs configured in the underlying config file.
- btd.definedFields.Add("Outputs")
-
- for _, glob := range task.Outputs {
- if strings.HasPrefix(glob, "!") {
- if filepath.IsAbs(glob[1:]) {
- log.Printf("[WARNING] Using an absolute path in \"outputs\" (%v) will not work and will be an error in a future version", glob)
- }
- exclusions = append(exclusions, glob[1:])
- } else {
- if filepath.IsAbs(glob) {
- log.Printf("[WARNING] Using an absolute path in \"outputs\" (%v) will not work and will be an error in a future version", glob)
- }
- inclusions = append(inclusions, glob)
- }
- }
-
- btd.TaskDefinition.Outputs = TaskOutputs{
- Inclusions: inclusions,
- Exclusions: exclusions,
- }
-
- sort.Strings(btd.TaskDefinition.Outputs.Inclusions)
- sort.Strings(btd.TaskDefinition.Outputs.Exclusions)
- }
-
- if task.Cache == nil {
- btd.TaskDefinition.ShouldCache = true
- } else {
- btd.definedFields.Add("ShouldCache")
- btd.TaskDefinition.ShouldCache = *task.Cache
- }
-
- envVarDependencies := make(util.Set)
- envVarPassthroughs := make(util.Set)
-
- btd.TaskDefinition.TopologicalDependencies = []string{} // TODO @mehulkar: this should be a set
- btd.TaskDefinition.TaskDependencies = []string{} // TODO @mehulkar: this should be a set
-
- // If there was a dependsOn field, add the bookkeeping
- // we don't care what's in the field, just that it was there
- // We'll use this marker to overwrite while merging TaskDefinitions.
- if task.DependsOn != nil {
- btd.definedFields.Add("DependsOn")
- }
-
- for _, dependency := range task.DependsOn {
- if strings.HasPrefix(dependency, envPipelineDelimiter) {
- log.Printf("[DEPRECATED] Declaring an environment variable in \"dependsOn\" is deprecated, found %s. Use the \"env\" key or use `npx @turbo/codemod migrate-env-var-dependencies`.\n", dependency)
- envVarDependencies.Add(strings.TrimPrefix(dependency, envPipelineDelimiter))
- } else if strings.HasPrefix(dependency, topologicalPipelineDelimiter) {
- // Note: This will get assigned multiple times in the loop, but we only care that it's true
- btd.TaskDefinition.TopologicalDependencies = append(btd.TaskDefinition.TopologicalDependencies, strings.TrimPrefix(dependency, topologicalPipelineDelimiter))
- } else {
- btd.TaskDefinition.TaskDependencies = append(btd.TaskDefinition.TaskDependencies, dependency)
- }
- }
-
- sort.Strings(btd.TaskDefinition.TaskDependencies)
- sort.Strings(btd.TaskDefinition.TopologicalDependencies)
-
- // Append env key into EnvVarDependencies
- if task.Env != nil {
- btd.definedFields.Add("EnvVarDependencies")
- if err := gatherEnvVars(task.Env, "env", &envVarDependencies); err != nil {
- return err
- }
- }
-
- btd.TaskDefinition.EnvVarDependencies = envVarDependencies.UnsafeListOfStrings()
-
- sort.Strings(btd.TaskDefinition.EnvVarDependencies)
-
- if task.PassthroughEnv != nil {
- btd.experimentalFields.Add("PassthroughEnv")
- if err := gatherEnvVars(task.PassthroughEnv, "passthrougEnv", &envVarPassthroughs); err != nil {
- return err
- }
- }
-
- btd.experimental.PassthroughEnv = envVarPassthroughs.UnsafeListOfStrings()
- sort.Strings(btd.experimental.PassthroughEnv)
-
- if task.Inputs != nil {
- // Note that we don't require Inputs to be sorted, we're going to
- // hash the resulting files and sort that instead
- btd.definedFields.Add("Inputs")
- // TODO: during rust port, this should be moved to a post-parse validation step
- for _, input := range task.Inputs {
- if filepath.IsAbs(input) {
- log.Printf("[WARNING] Using an absolute path in \"inputs\" (%v) will not work and will be an error in a future version", input)
- }
- }
- btd.TaskDefinition.Inputs = task.Inputs
- }
-
- if task.OutputMode != nil {
- btd.definedFields.Add("OutputMode")
- btd.TaskDefinition.OutputMode = *task.OutputMode
- }
-
- if task.Persistent != nil {
- btd.definedFields.Add("Persistent")
- btd.TaskDefinition.Persistent = *task.Persistent
- } else {
- btd.TaskDefinition.Persistent = false
- }
- return nil
-}
-
-// MarshalJSON serializes taskDefinitionHashable struct into json
-func (c taskDefinitionHashable) MarshalJSON() ([]byte, error) {
- task := makeRawTask(
- c.Persistent,
- c.ShouldCache,
- c.OutputMode,
- c.Inputs,
- c.Outputs,
- c.EnvVarDependencies,
- c.TaskDependencies,
- c.TopologicalDependencies,
- )
- return json.Marshal(task)
-}
-
-// MarshalJSON serializes TaskDefinition struct into json
-func (c TaskDefinition) MarshalJSON() ([]byte, error) {
- task := makeRawTask(
- c.Persistent,
- c.ShouldCache,
- c.OutputMode,
- c.Inputs,
- c.Outputs,
- c.EnvVarDependencies,
- c.TaskDependencies,
- c.TopologicalDependencies,
- )
-
- if len(c.PassthroughEnv) > 0 {
- task.PassthroughEnv = append(task.PassthroughEnv, c.PassthroughEnv...)
- }
- sort.Strings(task.PassthroughEnv)
-
- return json.Marshal(task)
-}
-
-// UnmarshalJSON deserializes the contents of turbo.json into a TurboJSON struct
-func (c *TurboJSON) UnmarshalJSON(data []byte) error {
- raw := &rawTurboJSON{}
- if err := json.Unmarshal(data, &raw); err != nil {
- return err
- }
-
- envVarDependencies := make(util.Set)
- envVarPassthroughs := make(util.Set)
- globalFileDependencies := make(util.Set)
-
- if err := gatherEnvVars(raw.GlobalEnv, "globalEnv", &envVarDependencies); err != nil {
- return err
- }
- if err := gatherEnvVars(raw.GlobalPassthroughEnv, "experimentalGlobalPassThroughEnv", &envVarPassthroughs); err != nil {
- return err
- }
-
- // TODO: In the rust port, warnings should be refactored to a post-parse validation step
- for _, value := range raw.GlobalDependencies {
- if strings.HasPrefix(value, envPipelineDelimiter) {
- log.Printf("[DEPRECATED] Declaring an environment variable in \"globalDependencies\" is deprecated, found %s. Use the \"globalEnv\" key or use `npx @turbo/codemod migrate-env-var-dependencies`.\n", value)
- envVarDependencies.Add(strings.TrimPrefix(value, envPipelineDelimiter))
- } else {
- if filepath.IsAbs(value) {
- log.Printf("[WARNING] Using an absolute path in \"globalDependencies\" (%v) will not work and will be an error in a future version", value)
- }
- globalFileDependencies.Add(value)
- }
- }
-
- // turn the set into an array and assign to the TurboJSON struct fields.
- c.GlobalEnv = envVarDependencies.UnsafeListOfStrings()
- sort.Strings(c.GlobalEnv)
-
- if raw.GlobalPassthroughEnv != nil {
- c.GlobalPassthroughEnv = envVarPassthroughs.UnsafeListOfStrings()
- sort.Strings(c.GlobalPassthroughEnv)
- }
-
- c.GlobalDeps = globalFileDependencies.UnsafeListOfStrings()
- sort.Strings(c.GlobalDeps)
-
- // copy these over, we don't need any changes here.
- c.Pipeline = raw.Pipeline
- c.RemoteCacheOptions = raw.RemoteCacheOptions
- c.Extends = raw.Extends
-
- return nil
-}
-
-// MarshalJSON converts a TurboJSON into the equivalent json object in bytes
-// note: we go via rawTurboJSON so that the output format is correct.
-// This is used by `turbo prune` to generate a pruned turbo.json
-// and also by --summarize & --dry=json to serialize the known config
-// into something we can print to screen
-func (c *TurboJSON) MarshalJSON() ([]byte, error) {
- raw := pristineTurboJSON{}
- raw.GlobalDependencies = c.GlobalDeps
- raw.GlobalEnv = c.GlobalEnv
- raw.GlobalPassthroughEnv = c.GlobalPassthroughEnv
- raw.Pipeline = c.Pipeline.Pristine()
- raw.RemoteCacheOptions = c.RemoteCacheOptions
-
- return json.Marshal(&raw)
-}
-
-func makeRawTask(persistent bool, shouldCache bool, outputMode util.TaskOutputMode, inputs []string, outputs TaskOutputs, envVarDependencies []string, taskDependencies []string, topologicalDependencies []string) *rawTaskWithDefaults {
- // Initialize with empty arrays, so we get empty arrays serialized into JSON
- task := &rawTaskWithDefaults{
- Outputs: []string{},
- Inputs: []string{},
- Env: []string{},
- PassthroughEnv: []string{},
- DependsOn: []string{},
- }
-
- task.Persistent = persistent
- task.Cache = &shouldCache
- task.OutputMode = outputMode
-
- if len(inputs) > 0 {
- task.Inputs = inputs
- }
-
- if len(envVarDependencies) > 0 {
- task.Env = append(task.Env, envVarDependencies...)
- }
-
- if len(outputs.Inclusions) > 0 {
- task.Outputs = append(task.Outputs, outputs.Inclusions...)
- }
-
- for _, i := range outputs.Exclusions {
- task.Outputs = append(task.Outputs, "!"+i)
- }
-
- if len(taskDependencies) > 0 {
- task.DependsOn = append(task.DependsOn, taskDependencies...)
- }
-
- for _, i := range topologicalDependencies {
- task.DependsOn = append(task.DependsOn, "^"+i)
- }
-
- // These _should_ already be sorted when the TaskDefinition struct was unmarshaled,
- // but we want to ensure they're sorted on the way out also, just in case something
- // in the middle mutates the items.
- sort.Strings(task.DependsOn)
- sort.Strings(task.Outputs)
- sort.Strings(task.Env)
- sort.Strings(task.Inputs)
- return task
-}
-
-// gatherEnvVars puts env vars into the provided set as long as they don't have an invalid value.
-func gatherEnvVars(vars []string, key string, into *util.Set) error {
- for _, value := range vars {
- if strings.HasPrefix(value, envPipelineDelimiter) {
- // Hard error to help people specify this correctly during migration.
- // TODO: Remove this error after we have run summary.
- return fmt.Errorf("You specified \"%s\" in the \"%s\" key. You should not prefix your environment variables with \"%s\"", value, key, envPipelineDelimiter)
- }
-
- into.Add(value)
- }
-
- return nil
-}
diff --git a/cli/internal/fs/turbo_json_test.go b/cli/internal/fs/turbo_json_test.go
deleted file mode 100644
index 1d384d5..0000000
--- a/cli/internal/fs/turbo_json_test.go
+++ /dev/null
@@ -1,277 +0,0 @@
-package fs
-
-import (
- "os"
- "reflect"
- "sort"
- "strings"
- "testing"
-
- "github.com/stretchr/testify/assert"
- "github.com/vercel/turbo/cli/internal/turbopath"
- "github.com/vercel/turbo/cli/internal/util"
- "gotest.tools/v3/assert/cmp"
-)
-
-func assertIsSorted(t *testing.T, arr []string, msg string) {
- t.Helper()
- if arr == nil {
- return
- }
-
- copied := make([]string, len(arr))
- copy(copied, arr)
- sort.Strings(copied)
- if !reflect.DeepEqual(arr, copied) {
- t.Errorf("Expected sorted, got %v: %v", arr, msg)
- }
-}
-
-func Test_ReadTurboConfig(t *testing.T) {
- testDir := getTestDir(t, "correct")
- turboJSON, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json"))
-
- if turboJSONReadErr != nil {
- t.Fatalf("invalid parse: %#v", turboJSONReadErr)
- }
-
- assert.EqualValues(t, []string{"AWS_SECRET_KEY"}, turboJSON.GlobalPassthroughEnv)
-
- pipelineExpected := map[string]BookkeepingTaskDefinition{
- "build": {
- definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "DependsOn"}),
- experimentalFields: util.SetFromStrings([]string{"PassthroughEnv"}),
- experimental: taskDefinitionExperiments{
- PassthroughEnv: []string{"GITHUB_TOKEN"},
- },
- TaskDefinition: taskDefinitionHashable{
- Outputs: TaskOutputs{Inclusions: []string{".next/**", "dist/**"}, Exclusions: []string{"dist/assets/**"}},
- TopologicalDependencies: []string{"build"},
- EnvVarDependencies: []string{},
- TaskDependencies: []string{},
- ShouldCache: true,
- OutputMode: util.NewTaskOutput,
- },
- },
- "lint": {
- definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "ShouldCache", "DependsOn"}),
- experimentalFields: util.SetFromStrings([]string{}),
- experimental: taskDefinitionExperiments{
- PassthroughEnv: []string{},
- },
- TaskDefinition: taskDefinitionHashable{
- Outputs: TaskOutputs{},
- TopologicalDependencies: []string{},
- EnvVarDependencies: []string{"MY_VAR"},
- TaskDependencies: []string{},
- ShouldCache: true,
- OutputMode: util.NewTaskOutput,
- },
- },
- "dev": {
- definedFields: util.SetFromStrings([]string{"OutputMode", "ShouldCache"}),
- experimentalFields: util.SetFromStrings([]string{}),
- experimental: taskDefinitionExperiments{
- PassthroughEnv: []string{},
- },
- TaskDefinition: taskDefinitionHashable{
- Outputs: TaskOutputs{},
- TopologicalDependencies: []string{},
- EnvVarDependencies: []string{},
- TaskDependencies: []string{},
- ShouldCache: false,
- OutputMode: util.FullTaskOutput,
- },
- },
- "publish": {
- definedFields: util.SetFromStrings([]string{"Inputs", "Outputs", "DependsOn", "ShouldCache"}),
- experimentalFields: util.SetFromStrings([]string{}),
- experimental: taskDefinitionExperiments{
- PassthroughEnv: []string{},
- },
- TaskDefinition: taskDefinitionHashable{
- Outputs: TaskOutputs{Inclusions: []string{"dist/**"}},
- TopologicalDependencies: []string{"build", "publish"},
- EnvVarDependencies: []string{},
- TaskDependencies: []string{"admin#lint", "build"},
- ShouldCache: false,
- Inputs: []string{"build/**/*"},
- OutputMode: util.FullTaskOutput,
- },
- },
- }
-
- validateOutput(t, turboJSON, pipelineExpected)
- remoteCacheOptionsExpected := RemoteCacheOptions{"team_id", true}
- assert.EqualValues(t, remoteCacheOptionsExpected, turboJSON.RemoteCacheOptions)
-}
-
-func Test_LoadTurboConfig_Legacy(t *testing.T) {
- testDir := getTestDir(t, "legacy-only")
- packageJSONPath := testDir.UntypedJoin("package.json")
- rootPackageJSON, pkgJSONReadErr := ReadPackageJSON(packageJSONPath)
-
- if pkgJSONReadErr != nil {
- t.Fatalf("invalid parse: %#v", pkgJSONReadErr)
- }
-
- _, turboJSONReadErr := LoadTurboConfig(testDir, rootPackageJSON, false)
- expectedErrorMsg := "Could not find turbo.json. Follow directions at https://turbo.build/repo/docs to create one: file does not exist"
- assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr)
-}
-
-func Test_LoadTurboConfig_BothCorrectAndLegacy(t *testing.T) {
- testDir := getTestDir(t, "both")
-
- packageJSONPath := testDir.UntypedJoin("package.json")
- rootPackageJSON, pkgJSONReadErr := ReadPackageJSON(packageJSONPath)
-
- if pkgJSONReadErr != nil {
- t.Fatalf("invalid parse: %#v", pkgJSONReadErr)
- }
-
- turboJSON, turboJSONReadErr := LoadTurboConfig(testDir, rootPackageJSON, false)
-
- if turboJSONReadErr != nil {
- t.Fatalf("invalid parse: %#v", turboJSONReadErr)
- }
-
- pipelineExpected := map[string]BookkeepingTaskDefinition{
- "build": {
- definedFields: util.SetFromStrings([]string{"Outputs", "OutputMode", "DependsOn"}),
- experimentalFields: util.SetFromStrings([]string{}),
- experimental: taskDefinitionExperiments{
- PassthroughEnv: []string{},
- },
- TaskDefinition: taskDefinitionHashable{
- Outputs: TaskOutputs{Inclusions: []string{".next/**", "dist/**"}, Exclusions: []string{"dist/assets/**"}},
- TopologicalDependencies: []string{"build"},
- EnvVarDependencies: []string{},
- TaskDependencies: []string{},
- ShouldCache: true,
- OutputMode: util.NewTaskOutput,
- },
- },
- }
-
- validateOutput(t, turboJSON, pipelineExpected)
-
- remoteCacheOptionsExpected := RemoteCacheOptions{"team_id", true}
- assert.EqualValues(t, remoteCacheOptionsExpected, turboJSON.RemoteCacheOptions)
- assert.Equal(t, rootPackageJSON.LegacyTurboConfig == nil, true)
-}
-
-func Test_ReadTurboConfig_InvalidEnvDeclarations1(t *testing.T) {
- testDir := getTestDir(t, "invalid-env-1")
- _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json"))
-
- expectedErrorMsg := "turbo.json: You specified \"$A\" in the \"env\" key. You should not prefix your environment variables with \"$\""
- assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr)
-}
-
-func Test_ReadTurboConfig_InvalidEnvDeclarations2(t *testing.T) {
- testDir := getTestDir(t, "invalid-env-2")
- _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json"))
- expectedErrorMsg := "turbo.json: You specified \"$A\" in the \"env\" key. You should not prefix your environment variables with \"$\""
- assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr)
-}
-
-func Test_ReadTurboConfig_InvalidGlobalEnvDeclarations(t *testing.T) {
- testDir := getTestDir(t, "invalid-global-env")
- _, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json"))
- expectedErrorMsg := "turbo.json: You specified \"$QUX\" in the \"globalEnv\" key. You should not prefix your environment variables with \"$\""
- assert.EqualErrorf(t, turboJSONReadErr, expectedErrorMsg, "Error should be: %v, got: %v", expectedErrorMsg, turboJSONReadErr)
-}
-
-func Test_ReadTurboConfig_EnvDeclarations(t *testing.T) {
- testDir := getTestDir(t, "legacy-env")
- turboJSON, turboJSONReadErr := readTurboConfig(testDir.UntypedJoin("turbo.json"))
-
- if turboJSONReadErr != nil {
- t.Fatalf("invalid parse: %#v", turboJSONReadErr)
- }
-
- pipeline := turboJSON.Pipeline
- assert.EqualValues(t, pipeline["task1"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"}))
- assert.EqualValues(t, pipeline["task2"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"}))
- assert.EqualValues(t, pipeline["task3"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"}))
- assert.EqualValues(t, pipeline["task4"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B"}))
- assert.EqualValues(t, pipeline["task6"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C", "D", "E", "F"}))
- assert.EqualValues(t, pipeline["task7"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C"}))
- assert.EqualValues(t, pipeline["task8"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B", "C"}))
- assert.EqualValues(t, pipeline["task9"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"}))
- assert.EqualValues(t, pipeline["task10"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A"}))
- assert.EqualValues(t, pipeline["task11"].TaskDefinition.EnvVarDependencies, sortedArray([]string{"A", "B"}))
-
- // check global env vars also
- assert.EqualValues(t, sortedArray([]string{"FOO", "BAR", "BAZ", "QUX"}), sortedArray(turboJSON.GlobalEnv))
- assert.EqualValues(t, sortedArray([]string{"somefile.txt"}), sortedArray(turboJSON.GlobalDeps))
-}
-
-func Test_TaskOutputsSort(t *testing.T) {
- inclusions := []string{"foo/**", "bar"}
- exclusions := []string{"special-file", ".hidden/**"}
- taskOutputs := TaskOutputs{Inclusions: inclusions, Exclusions: exclusions}
- sortedOutputs := taskOutputs.Sort()
- assertIsSorted(t, sortedOutputs.Inclusions, "Inclusions")
- assertIsSorted(t, sortedOutputs.Exclusions, "Exclusions")
- assert.False(t, cmp.DeepEqual(taskOutputs, sortedOutputs)().Success())
-}
-
-// Helpers
-func validateOutput(t *testing.T, turboJSON *TurboJSON, expectedPipeline Pipeline) {
- t.Helper()
- assertIsSorted(t, turboJSON.GlobalDeps, "Global Deps")
- assertIsSorted(t, turboJSON.GlobalEnv, "Global Env")
- validatePipeline(t, turboJSON.Pipeline, expectedPipeline)
-}
-
-func validatePipeline(t *testing.T, actual Pipeline, expected Pipeline) {
- t.Helper()
- // check top level keys
- if len(actual) != len(expected) {
- expectedKeys := []string{}
- for k := range expected {
- expectedKeys = append(expectedKeys, k)
- }
- actualKeys := []string{}
- for k := range actual {
- actualKeys = append(actualKeys, k)
- }
- t.Errorf("pipeline tasks mismatch. got %v, want %v", strings.Join(actualKeys, ","), strings.Join(expectedKeys, ","))
- }
-
- // check individual task definitions
- for taskName, expectedTaskDefinition := range expected {
- bookkeepingTaskDef, ok := actual[taskName]
- if !ok {
- t.Errorf("missing expected task: %v", taskName)
- }
- actualTaskDefinition := bookkeepingTaskDef.GetTaskDefinition()
- assertIsSorted(t, actualTaskDefinition.Outputs.Inclusions, "Task output inclusions")
- assertIsSorted(t, actualTaskDefinition.Outputs.Exclusions, "Task output exclusions")
- assertIsSorted(t, actualTaskDefinition.EnvVarDependencies, "Task env vars")
- assertIsSorted(t, actualTaskDefinition.PassthroughEnv, "Task env vars")
- assertIsSorted(t, actualTaskDefinition.TopologicalDependencies, "Topo deps")
- assertIsSorted(t, actualTaskDefinition.TaskDependencies, "Task deps")
- assert.EqualValuesf(t, expectedTaskDefinition, bookkeepingTaskDef, "task definition mismatch for %v", taskName)
- }
-}
-
-func getTestDir(t *testing.T, testName string) turbopath.AbsoluteSystemPath {
- defaultCwd, err := os.Getwd()
- if err != nil {
- t.Errorf("failed to get cwd: %v", err)
- }
- cwd, err := CheckedToAbsoluteSystemPath(defaultCwd)
- if err != nil {
- t.Fatalf("cwd is not an absolute directory %v: %v", defaultCwd, err)
- }
-
- return cwd.UntypedJoin("testdata", testName)
-}
-
-func sortedArray(arr []string) []string {
- sort.Strings(arr)
- return arr
-}