From 701c28ed770d619d8bc3cdcf6c8f7a9d698e06f6 Mon Sep 17 00:00:00 2001 From: 简律纯 Date: Tue, 2 May 2023 23:45:59 +0800 Subject: --- benchmark/.gitignore | 3 - benchmark/README.md | 6 -- benchmark/package.json | 16 --- benchmark/src/index.ts | 273 ------------------------------------------------- 4 files changed, 298 deletions(-) delete mode 100644 benchmark/.gitignore delete mode 100644 benchmark/README.md delete mode 100644 benchmark/package.json delete mode 100644 benchmark/src/index.ts diff --git a/benchmark/.gitignore b/benchmark/.gitignore deleted file mode 100644 index 956951a..0000000 --- a/benchmark/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -large-monorepo -benchmarks.json -tinybird.ndjson diff --git a/benchmark/README.md b/benchmark/README.md deleted file mode 100644 index 2f13f23..0000000 --- a/benchmark/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Turborepo Benchmarking - -To run benchmarks for turborepo - -1. From `../cli/` run `make turbo-prod` to build turbo -2. From this directory `pnpm run benchmark` diff --git a/benchmark/package.json b/benchmark/package.json deleted file mode 100644 index 9a92660..0000000 --- a/benchmark/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name": "benchmark", - "version": "1.0.0", - "dependencies": { - "esbuild": "^0.15.0", - "esbuild-register": "^3.3.2", - "fs-extra": "^10.0.0", - "ndjson": "^2.0.0" - }, - "scripts": { - "benchmark": "node -r esbuild-register src/index.ts" - }, - "devDependencies": { - "@types/node": "^16.11.49" - } -} diff --git a/benchmark/src/index.ts b/benchmark/src/index.ts deleted file mode 100644 index 117d6b1..0000000 --- a/benchmark/src/index.ts +++ /dev/null @@ -1,273 +0,0 @@ -import cp from "child_process"; -import fs from "fs"; -import fse from "fs-extra"; -import path from "path"; -import ndjson from "ndjson"; - -const REPO_ROOT = "large-monorepo"; -const REPO_ORIGIN = "https://github.com/gsoltis/large-monorepo.git"; -const REPO_PATH = path.join(process.cwd(), REPO_ROOT); -const REPETITIONS = 5; - -const DEFAULT_EXEC_OPTS = { stdio: "ignore" as const, cwd: REPO_PATH }; -const TURBO_BIN = path.resolve(path.join("..", "target", "release", "turbo")); -const DEFAULT_CACHE_PATH = path.join( - REPO_PATH, - "node_modules", - ".cache", - "turbo" -); -const ALT_CACHE_PATH = path.join( - REPO_PATH, - "node_modules", - ".cache", - "turbo-benchmark" -); - -type Timing = number; - -type Benchmark = { - name: string; - unit: string; - value: number; - range?: string; - extra?: string; -}; - -type TBirdEvent = { - commitSha: string; - commitTimestamp: Date; - platform: string; - benchmark: string; - durationMs: number; -}; - -function setup(): void { - // Clone repo if it doesn't exist, run clean - if (fs.existsSync(REPO_ROOT)) { - // reset the repo, remove all changed or untracked files - cp.execSync( - `cd ${REPO_ROOT} && git reset --hard HEAD && git clean -f -d -X`, - { - stdio: "inherit", - } - ); - } else { - cp.execSync(`git clone ${REPO_ORIGIN}`, { stdio: "ignore" }); - } - - // Run install so we aren't benchmarking node_modules ... - - cp.execSync("yarn install", DEFAULT_EXEC_OPTS); -} - -function cleanTurboCache(): void { - if (fs.existsSync(DEFAULT_CACHE_PATH)) { - console.log("clearing cache"); - fs.rmSync(DEFAULT_CACHE_PATH, { recursive: true }); - } -} - -function cleanBuild(): Timing[] { - const timings: Timing[] = []; - const isLocal = process.argv[process.argv.length - 1] == "--local"; - // We aren't really benchmarking this one, it OOMs if run in full parallel - // on GH actions - const repetitions = isLocal ? REPETITIONS : 1; - const concurrency = isLocal ? "" : " --concurrency=1"; - for (let i = 0; i < repetitions; i++) { - // clean first, we'll leave the cache in place for subsequent builds - cleanTurboCache(); - const start = new Date().getTime(); - cp.execSync(`${TURBO_BIN} run build${concurrency}`, DEFAULT_EXEC_OPTS); - const end = new Date().getTime(); - const timing = end - start; - timings.push(timing); - } - return timings; -} - -function cachedBuild(): Timing[] { - const timings: Timing[] = []; - for (let i = 0; i < REPETITIONS; i++) { - const start = new Date().getTime(); - cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS); - const end = new Date().getTime(); - const timing = end - start; - timings.push(timing); - } - return timings; -} - -function saveCache() { - // Remove any existing backup - if (fs.existsSync(ALT_CACHE_PATH)) { - fs.rmSync(ALT_CACHE_PATH, { recursive: true }); - } - // copy the current cache to the backup - if (fs.existsSync(DEFAULT_CACHE_PATH)) { - fse.copySync(DEFAULT_CACHE_PATH, ALT_CACHE_PATH, { recursive: true }); - } else { - // make an empty cache - fs.mkdirSync(ALT_CACHE_PATH, { recursive: true }); - } -} - -function restoreSavedCache() { - // Remove any existing cache - if (fs.existsSync(DEFAULT_CACHE_PATH)) { - fs.rmSync(DEFAULT_CACHE_PATH, { recursive: true }); - } - // Copy the backed-up cache to the real cache - fse.copySync(ALT_CACHE_PATH, DEFAULT_CACHE_PATH, { recursive: true }); -} - -function cachedBuildWithDelta(): Timing[] { - // Save existing cache just once, we'll restore from it each time - saveCache(); - - // Edit a file in place - const file = path.join( - REPO_PATH, - "packages", - "crew", - "important-feature-0", - "src", - "lib", - "important-component-0", - "important-component-0.tsx" - ); - const contents = fs.readFileSync(file).toString("utf-8"); - // make a small edit - const updated = contents.replace("-0!", "-0!!"); - fs.writeFileSync(file, updated); - - const timings: Timing[] = []; - for (let i = 0; i < REPETITIONS; i++) { - // Make sure we're starting with the cache from before we make the source code edit - restoreSavedCache(); - const start = new Date().getTime(); - cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS); - const end = new Date().getTime(); - const timing = end - start; - timings.push(timing); - } - return timings; -} - -function cachedBuildWithDependencyChange(): Timing[] { - // Save existing cache just once, we'll restore from it each time - saveCache(); - - // Edit a dependency - const file = path.join(REPO_PATH, "apps", "navigation", "package.json"); - const contents = JSON.parse(fs.readFileSync(file).toString("utf-8")); - contents.dependencies["crew-important-feature-0"] = "*"; - fs.writeFileSync(file, JSON.stringify(contents, null, 2)); - - const timings: Timing[] = []; - for (let i = 0; i < REPETITIONS; i++) { - // Make sure we're starting with the cache from before we made the dependency edit - restoreSavedCache(); - const start = new Date().getTime(); - cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS); - const end = new Date().getTime(); - const timing = end - start; - timings.push(timing); - } - return timings; -} - -class Benchmarks { - private readonly benchmarks: Benchmark[] = []; - private readonly tbirdEvents: TBirdEvent[] = []; - - constructor( - private readonly benchmarkFile: string, - private readonly tinybirdFile: string, - private readonly commitSha: string, - private readonly commitTimestamp: Date, - private readonly platform: string - ) {} - - run(name: string, b: () => Timing[]) { - console.log(name); - const timings = b(); - const max = Math.max(...timings); - const min = Math.min(...timings); - const avg = timings.reduce((a, b) => a + b, 0) / timings.length; - this.benchmarks.push({ - name, - value: avg, - unit: "ms", - range: String(max - min), - }); - timings.forEach((t) => { - this.tbirdEvents.push({ - commitSha: this.commitSha, - commitTimestamp: this.commitTimestamp, - platform: this.platform, - benchmark: name, - durationMs: t, - }); - }); - } - - flush() { - console.log(JSON.stringify(this.benchmarks, null, 2)); - fs.writeFileSync( - this.benchmarkFile, - JSON.stringify(this.benchmarks, null, 2) - ); - const stream = ndjson.stringify(); - const fd = fs.openSync(this.tinybirdFile, "w"); - stream.on("data", (line) => { - fs.writeSync(fd, line); - }); - this.tbirdEvents.forEach((t) => { - stream.write(t); - }); - stream.end(); - fs.closeSync(fd); - } -} - -cp.execSync(`${TURBO_BIN} --version`, { stdio: "inherit" }); - -function getCommitDetails(): { commitSha: string; commitTimestamp: Date } { - const envSha = process.env["GITHUB_SHA"]; - if (envSha === undefined) { - return { - commitSha: "unknown sha", - commitTimestamp: new Date(), - }; - } - const buf = cp.execSync(`git show -s --format=%ci ${envSha}`); - const dateString = String(buf).trim(); - const commitTimestamp = new Date(dateString); - return { - commitSha: envSha, - commitTimestamp, - }; -} - -const { commitSha, commitTimestamp } = getCommitDetails(); -const platform = process.env["RUNNER_OS"] ?? "unknown"; - -console.log("setup"); -setup(); -const benchmark = new Benchmarks( - "benchmarks.json", - "tinybird.ndjson", - commitSha, - commitTimestamp, - platform -); -benchmark.run("Clean Build", cleanBuild); -benchmark.run("Cached Build - No Change", cachedBuild); -benchmark.run("Cached Build - Code Change", cachedBuildWithDelta); -benchmark.run( - "Cached Build - Dependency Change", - cachedBuildWithDependencyChange -); -benchmark.flush(); -- cgit v1.2.3-70-g09d2