diff options
Diffstat (limited to 'cli/internal/client')
| -rw-r--r-- | cli/internal/client/analytics.go | 21 | ||||
| -rw-r--r-- | cli/internal/client/cache.go | 167 | ||||
| -rw-r--r-- | cli/internal/client/client.go | 309 | ||||
| -rw-r--r-- | cli/internal/client/client_test.go | 159 |
4 files changed, 0 insertions, 656 deletions
diff --git a/cli/internal/client/analytics.go b/cli/internal/client/analytics.go deleted file mode 100644 index 71381f0..0000000 --- a/cli/internal/client/analytics.go +++ /dev/null @@ -1,21 +0,0 @@ -package client - -import ( - "encoding/json" -) - -// RecordAnalyticsEvents is a specific method for POSTing events to Vercel -func (c *APIClient) RecordAnalyticsEvents(events []map[string]interface{}) error { - body, err := json.Marshal(events) - if err != nil { - return err - - } - - // We don't care about the response here - if _, err := c.JSONPost("/v8/artifacts/events", body); err != nil { - return err - } - - return nil -} diff --git a/cli/internal/client/cache.go b/cli/internal/client/cache.go deleted file mode 100644 index 11ad87a..0000000 --- a/cli/internal/client/cache.go +++ /dev/null @@ -1,167 +0,0 @@ -package client - -import ( - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "strings" - - "github.com/hashicorp/go-retryablehttp" - "github.com/vercel/turbo/cli/internal/ci" - "github.com/vercel/turbo/cli/internal/util" -) - -// PutArtifact uploads an artifact associated with a given hash string to the remote cache -func (c *APIClient) PutArtifact(hash string, artifactBody []byte, duration int, tag string) error { - if err := c.okToRequest(); err != nil { - return err - } - params := url.Values{} - c.addTeamParam(¶ms) - // only add a ? if it's actually needed (makes logging cleaner) - encoded := params.Encode() - if encoded != "" { - encoded = "?" + encoded - } - - requestURL := c.makeURL("/v8/artifacts/" + hash + encoded) - allowAuth := true - if c.usePreflight { - resp, latestRequestURL, err := c.doPreflight(requestURL, http.MethodPut, "Content-Type, x-artifact-duration, Authorization, User-Agent, x-artifact-tag") - if err != nil { - return fmt.Errorf("pre-flight request failed before trying to store in HTTP cache: %w", err) - } - requestURL = latestRequestURL - headers := resp.Header.Get("Access-Control-Allow-Headers") - allowAuth = strings.Contains(strings.ToLower(headers), strings.ToLower("Authorization")) - } - - req, err := retryablehttp.NewRequest(http.MethodPut, requestURL, artifactBody) - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("x-artifact-duration", fmt.Sprintf("%v", duration)) - if allowAuth { - req.Header.Set("Authorization", "Bearer "+c.token) - } - req.Header.Set("User-Agent", c.userAgent()) - if ci.IsCi() { - req.Header.Set("x-artifact-client-ci", ci.Constant()) - } - if tag != "" { - req.Header.Set("x-artifact-tag", tag) - } - if err != nil { - return fmt.Errorf("[WARNING] Invalid cache URL: %w", err) - } - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return fmt.Errorf("[ERROR] Failed to store files in HTTP cache: %w", err) - } - defer func() { _ = resp.Body.Close() }() - if resp.StatusCode == http.StatusForbidden { - return c.handle403(resp.Body) - } - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("[ERROR] Failed to store files in HTTP cache: %s against URL %s", resp.Status, requestURL) - } - return nil -} - -// FetchArtifact attempts to retrieve the build artifact with the given hash from the remote cache -func (c *APIClient) FetchArtifact(hash string) (*http.Response, error) { - return c.getArtifact(hash, http.MethodGet) -} - -// ArtifactExists attempts to determine if the build artifact with the given hash exists in the Remote Caching server -func (c *APIClient) ArtifactExists(hash string) (*http.Response, error) { - return c.getArtifact(hash, http.MethodHead) -} - -// getArtifact attempts to retrieve the build artifact with the given hash from the remote cache -func (c *APIClient) getArtifact(hash string, httpMethod string) (*http.Response, error) { - if httpMethod != http.MethodHead && httpMethod != http.MethodGet { - return nil, fmt.Errorf("invalid httpMethod %v, expected GET or HEAD", httpMethod) - } - - if err := c.okToRequest(); err != nil { - return nil, err - } - params := url.Values{} - c.addTeamParam(¶ms) - // only add a ? if it's actually needed (makes logging cleaner) - encoded := params.Encode() - if encoded != "" { - encoded = "?" + encoded - } - - requestURL := c.makeURL("/v8/artifacts/" + hash + encoded) - allowAuth := true - if c.usePreflight { - resp, latestRequestURL, err := c.doPreflight(requestURL, http.MethodGet, "Authorization, User-Agent") - if err != nil { - return nil, fmt.Errorf("pre-flight request failed before trying to fetch files in HTTP cache: %w", err) - } - requestURL = latestRequestURL - headers := resp.Header.Get("Access-Control-Allow-Headers") - allowAuth = strings.Contains(strings.ToLower(headers), strings.ToLower("Authorization")) - } - - req, err := retryablehttp.NewRequest(httpMethod, requestURL, nil) - if allowAuth { - req.Header.Set("Authorization", "Bearer "+c.token) - } - req.Header.Set("User-Agent", c.userAgent()) - if err != nil { - return nil, fmt.Errorf("invalid cache URL: %w", err) - } - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to fetch artifact: %v", err) - } else if resp.StatusCode == http.StatusForbidden { - err = c.handle403(resp.Body) - _ = resp.Body.Close() - return nil, err - } - return resp, nil -} - -func (c *APIClient) handle403(body io.Reader) error { - raw, err := ioutil.ReadAll(body) - if err != nil { - return fmt.Errorf("failed to read response %v", err) - } - apiError := &apiError{} - err = json.Unmarshal(raw, apiError) - if err != nil { - return fmt.Errorf("failed to read response (%v): %v", string(raw), err) - } - disabledErr, err := apiError.cacheDisabled() - if err != nil { - return err - } - return disabledErr -} - -type apiError struct { - Code string `json:"code"` - Message string `json:"message"` -} - -func (ae *apiError) cacheDisabled() (*util.CacheDisabledError, error) { - if strings.HasPrefix(ae.Code, "remote_caching_") { - statusString := ae.Code[len("remote_caching_"):] - status, err := util.CachingStatusFromString(statusString) - if err != nil { - return nil, err - } - return &util.CacheDisabledError{ - Status: status, - Message: ae.Message, - }, nil - } - return nil, fmt.Errorf("unknown status %v: %v", ae.Code, ae.Message) -} diff --git a/cli/internal/client/client.go b/cli/internal/client/client.go deleted file mode 100644 index 822b2df..0000000 --- a/cli/internal/client/client.go +++ /dev/null @@ -1,309 +0,0 @@ -// Package client implements some interfaces and convenience methods to interact with Vercel APIs and Remote Cache -package client - -import ( - "context" - "crypto/x509" - "errors" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "runtime" - "strings" - "sync/atomic" - "time" - - "github.com/hashicorp/go-hclog" - "github.com/hashicorp/go-retryablehttp" - "github.com/vercel/turbo/cli/internal/ci" -) - -// APIClient is the main interface for making network requests to Vercel -type APIClient struct { - // The api's base URL - baseURL string - token string - turboVersion string - - // Must be used via atomic package - currentFailCount uint64 - HTTPClient *retryablehttp.Client - teamID string - teamSlug string - // Whether or not to send preflight requests before uploads - usePreflight bool -} - -// ErrTooManyFailures is returned from remote cache API methods after `maxRemoteFailCount` errors have occurred -var ErrTooManyFailures = errors.New("skipping HTTP Request, too many failures have occurred") - -// _maxRemoteFailCount is the number of failed requests before we stop trying to upload/download -// artifacts to the remote cache -const _maxRemoteFailCount = uint64(3) - -// SetToken updates the APIClient's Token -func (c *APIClient) SetToken(token string) { - c.token = token -} - -// RemoteConfig holds the authentication and endpoint details for the API client -type RemoteConfig struct { - Token string - TeamID string - TeamSlug string - APIURL string -} - -// Opts holds values for configuring the behavior of the API client -type Opts struct { - UsePreflight bool - Timeout uint64 -} - -// ClientTimeout Exported ClientTimeout used in run.go -const ClientTimeout uint64 = 20 - -// NewClient creates a new APIClient -func NewClient(remoteConfig RemoteConfig, logger hclog.Logger, turboVersion string, opts Opts) *APIClient { - client := &APIClient{ - baseURL: remoteConfig.APIURL, - turboVersion: turboVersion, - HTTPClient: &retryablehttp.Client{ - HTTPClient: &http.Client{ - Timeout: time.Duration(opts.Timeout) * time.Second, - }, - RetryWaitMin: 2 * time.Second, - RetryWaitMax: 10 * time.Second, - RetryMax: 2, - Backoff: retryablehttp.DefaultBackoff, - Logger: logger, - }, - token: remoteConfig.Token, - teamID: remoteConfig.TeamID, - teamSlug: remoteConfig.TeamSlug, - usePreflight: opts.UsePreflight, - } - client.HTTPClient.CheckRetry = client.checkRetry - return client -} - -// hasUser returns true if we have credentials for a user -func (c *APIClient) hasUser() bool { - return c.token != "" -} - -// IsLinked returns true if we have a user and linked team -func (c *APIClient) IsLinked() bool { - return c.hasUser() && (c.teamID != "" || c.teamSlug != "") -} - -// GetTeamID returns the currently configured team id -func (c *APIClient) GetTeamID() string { - return c.teamID -} - -func (c *APIClient) retryCachePolicy(resp *http.Response, err error) (bool, error) { - if err != nil { - if errors.As(err, &x509.UnknownAuthorityError{}) { - // Don't retry if the error was due to TLS cert verification failure. - atomic.AddUint64(&c.currentFailCount, 1) - return false, err - } - atomic.AddUint64(&c.currentFailCount, 1) - return true, nil - } - - // 429 Too Many Requests is recoverable. Sometimes the server puts - // a Retry-After response header to indicate when the server is - // available to start processing request from client. - if resp.StatusCode == http.StatusTooManyRequests { - atomic.AddUint64(&c.currentFailCount, 1) - return true, nil - } - - // Check the response code. We retry on 500-range responses to allow - // the server time to recover, as 500's are typically not permanent - // errors and may relate to outages on the server side. This will catch - // invalid response codes as well, like 0 and 999. - if resp.StatusCode == 0 || (resp.StatusCode >= 500 && resp.StatusCode != 501) { - atomic.AddUint64(&c.currentFailCount, 1) - return true, fmt.Errorf("unexpected HTTP status %s", resp.Status) - } - - // swallow the error and stop retrying - return false, nil -} - -func (c *APIClient) checkRetry(ctx context.Context, resp *http.Response, err error) (bool, error) { - // do not retry on context.Canceled or context.DeadlineExceeded - if ctx.Err() != nil { - atomic.AddUint64(&c.currentFailCount, 1) - return false, ctx.Err() - } - - // we're squashing the error from the request and substituting any error that might come - // from our retry policy. - shouldRetry, err := c.retryCachePolicy(resp, err) - if shouldRetry { - // Our policy says it's ok to retry, but we need to check the failure count - if retryErr := c.okToRequest(); retryErr != nil { - return false, retryErr - } - } - return shouldRetry, err -} - -// okToRequest returns nil if it's ok to make a request, and returns the error to -// return to the caller if a request is not allowed -func (c *APIClient) okToRequest() error { - if atomic.LoadUint64(&c.currentFailCount) < _maxRemoteFailCount { - return nil - } - return ErrTooManyFailures -} - -func (c *APIClient) makeURL(endpoint string) string { - return fmt.Sprintf("%v%v", c.baseURL, endpoint) -} - -func (c *APIClient) userAgent() string { - return fmt.Sprintf("turbo %v %v %v (%v)", c.turboVersion, runtime.Version(), runtime.GOOS, runtime.GOARCH) -} - -// doPreflight returns response with closed body, latest request url, and any errors to the caller -func (c *APIClient) doPreflight(requestURL string, requestMethod string, requestHeaders string) (*http.Response, string, error) { - req, err := retryablehttp.NewRequest(http.MethodOptions, requestURL, nil) - req.Header.Set("User-Agent", c.userAgent()) - req.Header.Set("Access-Control-Request-Method", requestMethod) - req.Header.Set("Access-Control-Request-Headers", requestHeaders) - req.Header.Set("Authorization", "Bearer "+c.token) - if err != nil { - return nil, requestURL, fmt.Errorf("[WARNING] Invalid cache URL: %w", err) - } - - // If resp is not nil, ignore any errors - // because most likely unimportant for preflight to handle. - // Let follow-up request handle potential errors. - resp, err := c.HTTPClient.Do(req) - if resp == nil { - return resp, requestURL, err - } - defer resp.Body.Close() //nolint:golint,errcheck // nothing to do - // The client will continue following 307, 308 redirects until it hits - // max redirects, gets an error, or gets a normal response. - // Get the url from the Location header or get the url used in the last - // request (could have changed after following redirects). - // Note that net/http client does not continue redirecting the preflight - // request with the OPTIONS method for 301, 302, and 303 redirects. - // See golang/go Issue 18570. - if locationURL, err := resp.Location(); err == nil { - requestURL = locationURL.String() - } else { - requestURL = resp.Request.URL.String() - } - return resp, requestURL, nil -} - -func (c *APIClient) addTeamParam(params *url.Values) { - if c.teamID != "" && strings.HasPrefix(c.teamID, "team_") { - params.Add("teamId", c.teamID) - } - if c.teamSlug != "" { - params.Add("slug", c.teamSlug) - } -} - -// JSONPatch sends a byte array (json.marshalled payload) to a given endpoint with PATCH -func (c *APIClient) JSONPatch(endpoint string, body []byte) ([]byte, error) { - resp, err := c.request(endpoint, http.MethodPatch, body) - if err != nil { - return nil, err - } - - rawResponse, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response %v", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%s", string(rawResponse)) - } - - return rawResponse, nil -} - -// JSONPost sends a byte array (json.marshalled payload) to a given endpoint with POST -func (c *APIClient) JSONPost(endpoint string, body []byte) ([]byte, error) { - resp, err := c.request(endpoint, http.MethodPost, body) - if err != nil { - return nil, err - } - - rawResponse, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response %v", err) - } - - // For non 200/201 status codes, return the response body as an error - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("%s", string(rawResponse)) - } - - return rawResponse, nil -} - -func (c *APIClient) request(endpoint string, method string, body []byte) (*http.Response, error) { - if err := c.okToRequest(); err != nil { - return nil, err - } - - params := url.Values{} - c.addTeamParam(¶ms) - encoded := params.Encode() - if encoded != "" { - encoded = "?" + encoded - } - - requestURL := c.makeURL(endpoint + encoded) - - allowAuth := true - if c.usePreflight { - resp, latestRequestURL, err := c.doPreflight(requestURL, method, "Authorization, User-Agent") - if err != nil { - return nil, fmt.Errorf("pre-flight request failed before trying to fetch files in HTTP cache: %w", err) - } - - requestURL = latestRequestURL - headers := resp.Header.Get("Access-Control-Allow-Headers") - allowAuth = strings.Contains(strings.ToLower(headers), strings.ToLower("Authorization")) - } - - req, err := retryablehttp.NewRequest(method, requestURL, body) - if err != nil { - return nil, err - } - - // Set headers - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", c.userAgent()) - - if allowAuth { - req.Header.Set("Authorization", "Bearer "+c.token) - } - - if ci.IsCi() { - req.Header.Set("x-artifact-client-ci", ci.Constant()) - } - - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, err - } - - // If there isn't a response, something else probably went wrong - if resp == nil { - return nil, fmt.Errorf("response from %s is nil, something went wrong", requestURL) - } - - return resp, nil -} diff --git a/cli/internal/client/client_test.go b/cli/internal/client/client_test.go deleted file mode 100644 index 36ff3fb..0000000 --- a/cli/internal/client/client_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package client - -import ( - "bytes" - "encoding/json" - "errors" - "io/ioutil" - "net/http" - "net/http/httptest" - "reflect" - "testing" - - "github.com/google/uuid" - "github.com/hashicorp/go-hclog" - "github.com/vercel/turbo/cli/internal/util" -) - -func Test_sendToServer(t *testing.T) { - ch := make(chan []byte, 1) - ts := httptest.NewServer( - http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - b, err := ioutil.ReadAll(req.Body) - if err != nil { - t.Errorf("failed to read request %v", err) - } - ch <- b - w.WriteHeader(200) - w.Write([]byte{}) - })) - defer ts.Close() - - remoteConfig := RemoteConfig{ - TeamSlug: "my-team-slug", - APIURL: ts.URL, - Token: "my-token", - } - apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{}) - - myUUID, err := uuid.NewUUID() - if err != nil { - t.Errorf("failed to create uuid %v", err) - } - events := []map[string]interface{}{ - { - "sessionId": myUUID.String(), - "hash": "foo", - "source": "LOCAL", - "event": "hit", - }, - { - "sessionId": myUUID.String(), - "hash": "bar", - "source": "REMOTE", - "event": "MISS", - }, - } - - apiClient.RecordAnalyticsEvents(events) - - body := <-ch - - result := []map[string]interface{}{} - err = json.Unmarshal(body, &result) - if err != nil { - t.Errorf("unmarshalling body %v", err) - } - if !reflect.DeepEqual(events, result) { - t.Errorf("roundtrip got %v, want %v", result, events) - } -} - -func Test_PutArtifact(t *testing.T) { - ch := make(chan []byte, 1) - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - b, err := ioutil.ReadAll(req.Body) - if err != nil { - t.Errorf("failed to read request %v", err) - } - ch <- b - w.WriteHeader(200) - w.Write([]byte{}) - })) - defer ts.Close() - - // Set up test expected values - remoteConfig := RemoteConfig{ - TeamSlug: "my-team-slug", - APIURL: ts.URL, - Token: "my-token", - } - apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{}) - expectedArtifactBody := []byte("My string artifact") - - // Test Put Artifact - apiClient.PutArtifact("hash", expectedArtifactBody, 500, "") - testBody := <-ch - if !bytes.Equal(expectedArtifactBody, testBody) { - t.Errorf("Handler read '%v', wants '%v'", testBody, expectedArtifactBody) - } - -} - -func Test_PutWhenCachingDisabled(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer func() { _ = req.Body.Close() }() - w.WriteHeader(403) - _, _ = w.Write([]byte("{\"code\": \"remote_caching_disabled\",\"message\":\"caching disabled\"}")) - })) - defer ts.Close() - - // Set up test expected values - remoteConfig := RemoteConfig{ - TeamSlug: "my-team-slug", - APIURL: ts.URL, - Token: "my-token", - } - apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{}) - expectedArtifactBody := []byte("My string artifact") - // Test Put Artifact - err := apiClient.PutArtifact("hash", expectedArtifactBody, 500, "") - cd := &util.CacheDisabledError{} - if !errors.As(err, &cd) { - t.Errorf("expected cache disabled error, got %v", err) - } - if cd.Status != util.CachingStatusDisabled { - t.Errorf("caching status: expected %v, got %v", util.CachingStatusDisabled, cd.Status) - } -} - -func Test_FetchWhenCachingDisabled(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer func() { _ = req.Body.Close() }() - w.WriteHeader(403) - _, _ = w.Write([]byte("{\"code\": \"remote_caching_disabled\",\"message\":\"caching disabled\"}")) - })) - defer ts.Close() - - // Set up test expected values - remoteConfig := RemoteConfig{ - TeamSlug: "my-team-slug", - APIURL: ts.URL, - Token: "my-token", - } - apiClient := NewClient(remoteConfig, hclog.Default(), "v1", Opts{}) - // Test Put Artifact - resp, err := apiClient.FetchArtifact("hash") - cd := &util.CacheDisabledError{} - if !errors.As(err, &cd) { - t.Errorf("expected cache disabled error, got %v", err) - } - if cd.Status != util.CachingStatusDisabled { - t.Errorf("caching status: expected %v, got %v", util.CachingStatusDisabled, cd.Status) - } - if resp != nil { - t.Errorf("response got %v, want <nil>", resp) - } -} |
