aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/cli/internal/client
diff options
context:
space:
mode:
Diffstat (limited to 'cli/internal/client')
-rw-r--r--cli/internal/client/analytics.go21
-rw-r--r--cli/internal/client/cache.go167
-rw-r--r--cli/internal/client/client.go309
-rw-r--r--cli/internal/client/client_test.go159
4 files changed, 0 insertions, 656 deletions
diff --git a/cli/internal/client/analytics.go b/cli/internal/client/analytics.go
deleted file mode 100644
index 71381f0..0000000
--- a/cli/internal/client/analytics.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package client
-
-import (
- "encoding/json"
-)
-
-// RecordAnalyticsEvents is a specific method for POSTing events to Vercel
-func (c *APIClient) RecordAnalyticsEvents(events []map[string]interface{}) error {
- body, err := json.Marshal(events)
- if err != nil {
- return err
-
- }
-
- // We don't care about the response here
- if _, err := c.JSONPost("/v8/artifacts/events", body); err != nil {
- return err
- }
-
- return nil
-}
diff --git a/cli/internal/client/cache.go b/cli/internal/client/cache.go
deleted file mode 100644
index 11ad87a..0000000
--- a/cli/internal/client/cache.go
+++ /dev/null
@@ -1,167 +0,0 @@
-package client
-
-import (
- "encoding/json"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "net/url"
- "strings"
-
- "github.com/hashicorp/go-retryablehttp"
- "github.com/vercel/turbo/cli/internal/ci"
- "github.com/vercel/turbo/cli/internal/util"
-)
-
-// PutArtifact uploads an artifact associated with a given hash string to the remote cache
-func (c *APIClient) PutArtifact(hash string, artifactBody []byte, duration int, tag string) error {
- if err := c.okToRequest(); err != nil {
- return err
- }
- params := url.Values{}
- c.addTeamParam(&params)
- // only add a ? if it's actually needed (makes logging cleaner)
- encoded := params.Encode()
- if encoded != "" {
- encoded = "?" + encoded
- }
-
- requestURL := c.makeURL("/v8/artifacts/" + hash + encoded)
- allowAuth := true
- if c.usePreflight {
- resp, latestRequestURL, err := c.doPreflight(requestURL, http.MethodPut, "Content-Type, x-artifact-duration, Authorization, User-Agent, x-artifact-tag")
- if err != nil {
- return fmt.Errorf("pre-flight request failed before trying to store in HTTP cache: %w", err)
- }
- requestURL = latestRequestURL
- headers := resp.Header.Get("Access-Control-Allow-Headers")
- allowAuth = strings.Contains(strings.ToLower(headers), strings.ToLower("Authorization"))
- }
-
- req, err := retryablehttp.NewRequest(http.MethodPut, requestURL, artifactBody)
- req.Header.Set("Content-Type", "application/octet-stream")
- req.Header.Set("x-artifact-duration", fmt.Sprintf("%v", duration))
- if allowAuth {
- req.Header.Set("Authorization", "Bearer "+c.token)
- }
- req.Header.Set("User-Agent", c.userAgent())
- if ci.IsCi() {
- req.Header.Set("x-artifact-client-ci", ci.Constant())
- }
- if tag != "" {
- req.Header.Set("x-artifact-tag", tag)
- }
- if err != nil {
- return fmt.Errorf("[WARNING] Invalid cache URL: %w", err)
- }
-
- resp, err := c.HTTPClient.Do(req)
- if err != nil {
- return fmt.Errorf("[ERROR] Failed to store files in HTTP cache: %w", err)
- }
- defer func() { _ = resp.Body.Close() }()
- if resp.StatusCode == http.StatusForbidden {
- return c.handle403(resp.Body)
- }
- if resp.StatusCode != http.StatusOK {
- return fmt.Errorf("[ERROR] Failed to store files in HTTP cache: %s against URL %s", resp.Status, requestURL)
- }
- return nil
-}
-
-// FetchArtifact attempts to retrieve the build artifact with the given hash from the remote cache
-func (c *APIClient) FetchArtifact(hash string) (*http.Response, error) {
- return c.getArtifact(hash, http.MethodGet)
-}
-
-// ArtifactExists attempts to determine if the build artifact with the given hash exists in the Remote Caching server
-func (c *APIClient) ArtifactExists(hash string) (*http.Response, error) {
- return c.getArtifact(hash, http.MethodHead)
-}
-
-// getArtifact attempts to retrieve the build artifact with the given hash from the remote cache
-func (c *APIClient) getArtifact(hash string, httpMethod string) (*http.Response, error) {
- if httpMethod != http.MethodHead && httpMethod != http.MethodGet {
- return nil, fmt.Errorf("invalid httpMethod %v, expected GET or HEAD", httpMethod)
- }
-
- if err := c.okToRequest(); err != nil {
- return nil, err
- }
- params := url.Values{}
- c.addTeamParam(&params)
- // only add a ? if it's actually needed (makes logging cleaner)
- encoded := params.Encode()
- if encoded != "" {
- encoded = "?" + encoded
- }
-
- requestURL := c.makeURL("/v8/artifacts/" + hash + encoded)
- allowAuth := true
- if c.usePreflight {
- resp, latestRequestURL, err := c.doPreflight(requestURL, http.MethodGet, "Authorization, User-Agent")
- if err != nil {
- return nil, fmt.Errorf("pre-flight request failed before trying to fetch files in HTTP cache: %w", err)
- }
- requestURL = latestRequestURL
- headers := resp.Header.Get("Access-Control-Allow-Headers")
- allowAuth = strings.Contains(strings.ToLower(headers), strings.ToLower("Authorization"))
- }
-
- req, err := retryablehttp.NewRequest(httpMethod, requestURL, nil)
- if allowAuth {
- req.Header.Set("Authorization", "Bearer "+c.token)
- }
- req.Header.Set("User-Agent", c.userAgent())
- if err != nil {
- return nil, fmt.Errorf("invalid cache URL: %w", err)
- }
-
- resp, err := c.HTTPClient.Do(req)
- if err != nil {
- return nil, fmt.Errorf("failed to fetch artifact: %v", err)
- } else if resp.StatusCode == http.StatusForbidden {
- err = c.handle403(resp.Body)
- _ = resp.Body.Close()
- return nil, err
- }
- return resp, nil
-}
-
-func (c *APIClient) handle403(body io.Reader) error {
- raw, err := ioutil.ReadAll(body)
- if err != nil {
- return fmt.Errorf("failed to read response %v", err)
- }
- apiError := &apiError{}
- err = json.Unmarshal(raw, apiError)
- if err != nil {
- return fmt.Errorf("failed to read response (%v): %v", string(raw), err)
- }
- disabledErr, err := apiError.cacheDisabled()
- if err != nil {
- return err
- }
- return disabledErr
-}
-
-type apiError struct {
- Code string `json:"code"`
- Message string `json:"message"`
-}
-
-func (ae *apiError) cacheDisabled() (*util.CacheDisabledError, error) {
- if strings.HasPrefix(ae.Code, "remote_caching_") {
- statusString := ae.Code[len("remote_caching_"):]
- status, err := util.CachingStatusFromString(statusString)
- if err != nil {
- return nil, err
- }
- return &util.CacheDisabledError{
- Status: status,
- Message: ae.Message,
- }, nil
- }
- return nil, fmt.Errorf("unknown status %v: %v", ae.Code, ae.Message)
-}
diff --git a/cli/internal/client/client.go b/cli/internal/client/client.go
deleted file mode 100644
index 822b2df..0000000
--- a/cli/internal/client/client.go
+++ /dev/null
@@ -1,309 +0,0 @@
-// Package client implements some interfaces and convenience methods to interact with Vercel APIs and Remote Cache
-package client
-
-import (
- "context"
- "crypto/x509"
- "errors"
- "fmt"
- "io/ioutil"
- "net/http"
- "net/url"
- "runtime"
- "strings"
- "sync/atomic"
- "time"
-
- "github.com/hashicorp/go-hclog"
- "github.com/hashicorp/go-retryablehttp"
- "github.com/vercel/turbo/cli/internal/ci"
-)
-
-// APIClient is the main interface for making network requests to Vercel
-type APIClient struct {
- // The api's base URL
- baseURL string
- token string
- turboVersion string
-
- // Must be used via atomic package
- currentFailCount uint64
- HTTPClient *retryablehttp.Client
- teamID string
- teamSlug string
- // Whether or not to send preflight requests before uploads
- usePreflight bool
-}
-
-// ErrTooManyFailures is returned from remote cache API methods after `maxRemoteFailCount` errors have occurred
-var ErrTooManyFailures = errors.New("skipping HTTP Request, too many failures have occurred")
-
-// _maxRemoteFailCount is the number of failed requests before we stop trying to upload/download
-// artifacts to the remote cache
-const _maxRemoteFailCount = uint64(3)
-
-// SetToken updates the APIClient's Token
-func (c *APIClient) SetToken(token string) {
- c.token = token
-}
-
-// RemoteConfig holds the authentication and endpoint details for the API client
-type RemoteConfig struct {
- Token string
- TeamID string
- TeamSlug string
- APIURL string
-}
-
-// Opts holds values for configuring the behavior of the API client
-type Opts struct {
- UsePreflight bool
- Timeout uint64
-}
-
-// ClientTimeout Exported ClientTimeout used in run.go
-const ClientTimeout uint64 = 20
-
-// NewClient creates a new APIClient
-func NewClient(remoteConfig RemoteConfig, logger hclog.Logger, turboVersion string, opts Opts) *APIClient {
- client := &APIClient{
- baseURL: remoteConfig.APIURL,
- turboVersion: turboVersion,
- HTTPClient: &retryablehttp.Client{
- HTTPClient: &http.Client{
- Timeout: time.Duration(opts.Timeout) * time.Second,
- },
- RetryWaitMin: 2 * time.Second,
- RetryWaitMax: 10 * time.Second,
- RetryMax: 2,
- Backoff: retryablehttp.DefaultBackoff,
- Logger: logger,
- },
- token: remoteConfig.Token,
- teamID: remoteConfig.TeamID,
- teamSlug: remoteConfig.TeamSlug,
- usePreflight: opts.UsePreflight,
- }
- client.HTTPClient.CheckRetry = client.checkRetry
- return client
-}
-
-// hasUser returns true if we have credentials for a user
-func (c *APIClient) hasUser() bool {
- return c.token != ""
-}
-
-// IsLinked returns true if we have a user and linked team
-func (c *APIClient) IsLinked() bool {
- return c.hasUser() && (c.teamID != "" || c.teamSlug != "")
-}
-
-// GetTeamID returns the currently configured team id
-func (c *APIClient) GetTeamID() string {
- return c.teamID
-}
-
-func (c *APIClient) retryCachePolicy(resp *http.Response, err error) (bool, error) {
- if err != nil {
- if errors.As(err, &x509.UnknownAuthorityError{}) {
- // Don't retry if the error was due to TLS cert verification failure.
- atomic.AddUint64(&c.currentFailCount, 1)
- return false, err
- }
- atomic.AddUint64(&c.currentFailCount, 1)
- return true, nil
- }
-
- // 429 Too Many Requests is recoverable. Sometimes the server puts
- // a Retry-After response header to indicate when the server is
- // available to start processing request from client.
- if resp.StatusCode == http.StatusTooManyRequests {
- atomic.AddUint64(&c.currentFailCount, 1)
- return true, nil
- }
-
- // Check the response code. We retry on 500-range responses to allow
- // the server time to recover, as 500's are typically not permanent
- // errors and may relate to outages on the server side. This will catch
- // invalid response codes as well, like 0 and 999.
- if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != 501) {
- atomic.AddUint64(&c.currentFailCount, 1)
- return true, fmt.Errorf("unexpected HTTP status %s", resp.Status)
- }
-
- // swallow the error and stop retrying
- return false, nil
-}
-
-func (c *APIClient) checkRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
- // do not retry on context.Canceled or context.DeadlineExceeded
- if ctx.Err() != nil {
- atomic.AddUint64(&c.currentFailCount, 1)
- return false, ctx.Err()
- }
-
- // we're squashing the error from the request and substituting any error that might come
- // from our retry policy.
- shouldRetry, err := c.retryCachePolicy(resp, err)
- if shouldRetry {
- // Our policy says it's ok to retry, but we need to check the failure count
- if retryErr := c.okToRequest(); retryErr != nil {
- return false, retryErr
- }
- }
- return shouldRetry, err
-}
-
-// okToRequest returns nil if it's ok to make a request, and returns the error to
-// return to the caller if a request is not allowed
-func (c *APIClient) okToRequest() error {
- if atomic.LoadUint64(&c.currentFailCount) < _maxRemoteFailCount {
- return nil
- }
- return ErrTooManyFailures
-}
-
-func (c *APIClient) makeURL(endpoint string) string {
- return fmt.Sprintf("%v%v", c.baseURL, endpoint)
-}
-
-func (c *APIClient) userAgent() string {
- return fmt.Sprintf("turbo %v %v %v (%v)", c.turboVersion, runtime.Version(), runtime.GOOS, runtime.GOARCH)
-}
-
-// doPreflight returns response with closed body, latest request url, and any errors to the caller
-func (c *APIClient) doPreflight(requestURL string, requestMethod string, requestHeaders string) (*http.Response, string, error) {
- req, err := retryablehttp.NewRequest(http.MethodOptions, requestURL, nil)
- req.Header.Set("User-Agent", c.userAgent())
- req.Header.Set("Access-Control-Request-Method", requestMethod)
- req.Header.Set("Access-Control-Request-Headers", requestHeaders)
- req.Header.Set("Authorization", "Bearer "+c.token)
- if err != nil {
- return nil, requestURL, fmt.Errorf("[WARNING] Invalid cache URL: %w", err)
- }
-
- // If resp is not nil, ignore any errors
- // because most likely unimportant for preflight to handle.
- // Let follow-up request handle potential errors.
- resp, err := c.HTTPClient.Do(req)
- if resp == nil {
- return resp, requestURL, err
- }
- defer resp.Body.Close() //nolint:golint,errcheck // nothing to do
- // The client will continue following 307, 308 redirects until it hits
- // max redirects, gets an error, or gets a normal response.
- // Get the url from the Location header or get the url used in the last
- // request (could have changed after following redirects).
- // Note that net/http client does not continue redirecting the preflight
- // request with the OPTIONS method for 301, 302, and 303 redirects.
- // See golang/go Issue 18570.
- if locationURL, err := resp.Location(); err == nil {
- requestURL = locationURL.String()
- } else {
- requestURL = resp.Request.URL.String()
- }
- return resp, requestURL, nil
-}
-
-func (c *APIClient) addTeamParam(params *url.Values) {
- if c.teamID != "" && strings.HasPrefix(c.teamID, "team_") {
- params.Add("teamId", c.teamID)
- }
- if c.teamSlug != "" {
- params.Add("slug", c.teamSlug)
- }
-}
-
-// JSONPatch sends a byte array (json.marshalled payload) to a given endpoint with PATCH
-func (c *APIClient) JSONPatch(endpoint string, body []byte) ([]byte, error) {
- resp, err := c.request(endpoint, http.MethodPatch, body)
- if err != nil {
- return nil, err
- }
-
- rawResponse, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response %v", err)
- }
- if resp.StatusCode != http.StatusOK {
- return nil, fmt.Errorf("%s", string(rawResponse))
- }
-
- return rawResponse, nil
-}
-
-// JSONPost sends a byte array (json.marshalled payload) to a given endpoint with POST
-func (c *APIClient) JSONPost(endpoint string, body []byte) ([]byte, error) {
- resp, err := c.request(endpoint, http.MethodPost, body)
- if err != nil {
- return nil, err
- }
-
- rawResponse, err := ioutil.ReadAll(resp.Body)
- if err != nil {
- return nil, fmt.Errorf("failed to read response %v", err)
- }
-
- // For non 200/201 status codes, return the response body as an error
- if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
- return nil, fmt.Errorf("%s", string(rawResponse))
- }
-
- return rawResponse, nil
-}
-
-func (c *APIClient) request(endpoint string, method string, body []byte) (*http.Response, error) {
- if err := c.okToRequest(); err != nil {
- return nil, err
- }
-
- params := url.Values{}
- c.addTeamParam(&params)
- encoded := params.Encode()
- if encoded != "" {
- encoded = "?" + encoded
- }
-
- requestURL := c.makeURL(endpoint + encoded)
-
- allowAuth := true
- if c.usePreflight {
- resp, latestRequestURL, err := c.doPreflight(requestURL, method, "Authorization, User-Agent")
- if err != nil {
- return nil, fmt.Errorf("pre-flight request failed before trying to fetch files in HTTP cache: %w", err)
- }
-
- requestURL = latestRequestURL
- headers := resp.Header.Get("Access-Control-Allow-Headers")
- allowAuth = strings.Contains(strings.ToLower(headers), strings.ToLower("Authorization"))
- }
-
- req, err := retryablehttp.NewRequest(method, requestURL, body)
- if err != nil {
- return nil, err
- }
-
- // Set headers
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("User-Agent", c.userAgent())
-
- if allowAuth {
- req.Header.Set("Authorization", "Bearer "+c.token)
- }
-
- if ci.IsCi() {
- req.Header.Set("x-artifact-client-ci", ci.Constant())
- }
-
- resp, err := c.HTTPClient.Do(req)
- if err != nil {
- return nil, err
- }
-
- // If there isn't a response, something else probably went wrong
- if resp == nil {
- return nil, fmt.Errorf("response from %s is nil, something went wrong", requestURL)
- }
-
- return resp, nil
-}
diff --git a/cli/internal/client/client_test.go b/cli/internal/client/client_test.go
deleted file mode 100644
index 36ff3fb..0000000
--- a/cli/internal/client/client_test.go
+++ /dev/null
@@ -1,159 +0,0 @@
-package client
-
-import (
- "bytes"
- "encoding/json"
- "errors"
- "io/ioutil"
- "net/http"
- "net/http/httptest"
- "reflect"
- "testing"
-
- "github.com/google/uuid"
- "github.com/hashicorp/go-hclog"
- "github.com/vercel/turbo/cli/internal/util"
-)
-
-func Test_sendToServer(t *testing.T) {
- ch := make(chan []byte, 1)
- ts := httptest.NewServer(
- http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- defer req.Body.Close()
- b, err := ioutil.ReadAll(req.Body)
- if err != nil {
- t.Errorf("failed to read request %v", err)
- }
- ch <- b
- w.WriteHeader(200)
- w.Write([]byte{})
- }))
- defer ts.Close()
-
- remoteConfig := RemoteConfig{
- TeamSlug: "my-team-slug",
- APIURL: ts.URL,
- Token: "my-token",
- }
- apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{})
-
- myUUID, err := uuid.NewUUID()
- if err != nil {
- t.Errorf("failed to create uuid %v", err)
- }
- events := []map[string]interface{}{
- {
- "sessionId": myUUID.String(),
- "hash": "foo",
- "source": "LOCAL",
- "event": "hit",
- },
- {
- "sessionId": myUUID.String(),
- "hash": "bar",
- "source": "REMOTE",
- "event": "MISS",
- },
- }
-
- apiClient.RecordAnalyticsEvents(events)
-
- body := <-ch
-
- result := []map[string]interface{}{}
- err = json.Unmarshal(body, &result)
- if err != nil {
- t.Errorf("unmarshalling body %v", err)
- }
- if !reflect.DeepEqual(events, result) {
- t.Errorf("roundtrip got %v, want %v", result, events)
- }
-}
-
-func Test_PutArtifact(t *testing.T) {
- ch := make(chan []byte, 1)
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- defer req.Body.Close()
- b, err := ioutil.ReadAll(req.Body)
- if err != nil {
- t.Errorf("failed to read request %v", err)
- }
- ch <- b
- w.WriteHeader(200)
- w.Write([]byte{})
- }))
- defer ts.Close()
-
- // Set up test expected values
- remoteConfig := RemoteConfig{
- TeamSlug: "my-team-slug",
- APIURL: ts.URL,
- Token: "my-token",
- }
- apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{})
- expectedArtifactBody := []byte("My string artifact")
-
- // Test Put Artifact
- apiClient.PutArtifact("hash", expectedArtifactBody, 500, "")
- testBody := <-ch
- if !bytes.Equal(expectedArtifactBody, testBody) {
- t.Errorf("Handler read '%v', wants '%v'", testBody, expectedArtifactBody)
- }
-
-}
-
-func Test_PutWhenCachingDisabled(t *testing.T) {
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- defer func() { _ = req.Body.Close() }()
- w.WriteHeader(403)
- _, _ = w.Write([]byte("{\"code\": \"remote_caching_disabled\",\"message\":\"caching disabled\"}"))
- }))
- defer ts.Close()
-
- // Set up test expected values
- remoteConfig := RemoteConfig{
- TeamSlug: "my-team-slug",
- APIURL: ts.URL,
- Token: "my-token",
- }
- apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{})
- expectedArtifactBody := []byte("My string artifact")
- // Test Put Artifact
- err := apiClient.PutArtifact("hash", expectedArtifactBody, 500, "")
- cd := &util.CacheDisabledError{}
- if !errors.As(err, &cd) {
- t.Errorf("expected cache disabled error, got %v", err)
- }
- if cd.Status != util.CachingStatusDisabled {
- t.Errorf("caching status: expected %v, got %v", util.CachingStatusDisabled, cd.Status)
- }
-}
-
-func Test_FetchWhenCachingDisabled(t *testing.T) {
- ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
- defer func() { _ = req.Body.Close() }()
- w.WriteHeader(403)
- _, _ = w.Write([]byte("{\"code\": \"remote_caching_disabled\",\"message\":\"caching disabled\"}"))
- }))
- defer ts.Close()
-
- // Set up test expected values
- remoteConfig := RemoteConfig{
- TeamSlug: "my-team-slug",
- APIURL: ts.URL,
- Token: "my-token",
- }
- apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{})
- // Test Put Artifact
- resp, err := apiClient.FetchArtifact("hash")
- cd := &util.CacheDisabledError{}
- if !errors.As(err, &cd) {
- t.Errorf("expected cache disabled error, got %v", err)
- }
- if cd.Status != util.CachingStatusDisabled {
- t.Errorf("caching status: expected %v, got %v", util.CachingStatusDisabled, cd.Status)
- }
- if resp != nil {
- t.Errorf("response got %v, want <nil>", resp)
- }
-}