aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/benchmark
diff options
context:
space:
mode:
Diffstat (limited to 'benchmark')
-rw-r--r--benchmark/.gitignore3
-rw-r--r--benchmark/README.md6
-rw-r--r--benchmark/package.json16
-rw-r--r--benchmark/src/index.ts273
4 files changed, 0 insertions, 298 deletions
diff --git a/benchmark/.gitignore b/benchmark/.gitignore
deleted file mode 100644
index 956951a..0000000
--- a/benchmark/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-large-monorepo
-benchmarks.json
-tinybird.ndjson
diff --git a/benchmark/README.md b/benchmark/README.md
deleted file mode 100644
index 2f13f23..0000000
--- a/benchmark/README.md
+++ /dev/null
@@ -1,6 +0,0 @@
-# Turborepo Benchmarking
-
-To run benchmarks for turborepo
-
-1. From `../cli/` run `make turbo-prod` to build turbo
-2. From this directory `pnpm run benchmark`
diff --git a/benchmark/package.json b/benchmark/package.json
deleted file mode 100644
index 9a92660..0000000
--- a/benchmark/package.json
+++ /dev/null
@@ -1,16 +0,0 @@
-{
- "name": "benchmark",
- "version": "1.0.0",
- "dependencies": {
- "esbuild": "^0.15.0",
- "esbuild-register": "^3.3.2",
- "fs-extra": "^10.0.0",
- "ndjson": "^2.0.0"
- },
- "scripts": {
- "benchmark": "node -r esbuild-register src/index.ts"
- },
- "devDependencies": {
- "@types/node": "^16.11.49"
- }
-}
diff --git a/benchmark/src/index.ts b/benchmark/src/index.ts
deleted file mode 100644
index 117d6b1..0000000
--- a/benchmark/src/index.ts
+++ /dev/null
@@ -1,273 +0,0 @@
-import cp from "child_process";
-import fs from "fs";
-import fse from "fs-extra";
-import path from "path";
-import ndjson from "ndjson";
-
-const REPO_ROOT = "large-monorepo";
-const REPO_ORIGIN = "https://github.com/gsoltis/large-monorepo.git";
-const REPO_PATH = path.join(process.cwd(), REPO_ROOT);
-const REPETITIONS = 5;
-
-const DEFAULT_EXEC_OPTS = { stdio: "ignore" as const, cwd: REPO_PATH };
-const TURBO_BIN = path.resolve(path.join("..", "target", "release", "turbo"));
-const DEFAULT_CACHE_PATH = path.join(
- REPO_PATH,
- "node_modules",
- ".cache",
- "turbo"
-);
-const ALT_CACHE_PATH = path.join(
- REPO_PATH,
- "node_modules",
- ".cache",
- "turbo-benchmark"
-);
-
-type Timing = number;
-
-type Benchmark = {
- name: string;
- unit: string;
- value: number;
- range?: string;
- extra?: string;
-};
-
-type TBirdEvent = {
- commitSha: string;
- commitTimestamp: Date;
- platform: string;
- benchmark: string;
- durationMs: number;
-};
-
-function setup(): void {
- // Clone repo if it doesn't exist, run clean
- if (fs.existsSync(REPO_ROOT)) {
- // reset the repo, remove all changed or untracked files
- cp.execSync(
- `cd ${REPO_ROOT} && git reset --hard HEAD && git clean -f -d -X`,
- {
- stdio: "inherit",
- }
- );
- } else {
- cp.execSync(`git clone ${REPO_ORIGIN}`, { stdio: "ignore" });
- }
-
- // Run install so we aren't benchmarking node_modules ...
-
- cp.execSync("yarn install", DEFAULT_EXEC_OPTS);
-}
-
-function cleanTurboCache(): void {
- if (fs.existsSync(DEFAULT_CACHE_PATH)) {
- console.log("clearing cache");
- fs.rmSync(DEFAULT_CACHE_PATH, { recursive: true });
- }
-}
-
-function cleanBuild(): Timing[] {
- const timings: Timing[] = [];
- const isLocal = process.argv[process.argv.length - 1] == "--local";
- // We aren't really benchmarking this one, it OOMs if run in full parallel
- // on GH actions
- const repetitions = isLocal ? REPETITIONS : 1;
- const concurrency = isLocal ? "" : " --concurrency=1";
- for (let i = 0; i < repetitions; i++) {
- // clean first, we'll leave the cache in place for subsequent builds
- cleanTurboCache();
- const start = new Date().getTime();
- cp.execSync(`${TURBO_BIN} run build${concurrency}`, DEFAULT_EXEC_OPTS);
- const end = new Date().getTime();
- const timing = end - start;
- timings.push(timing);
- }
- return timings;
-}
-
-function cachedBuild(): Timing[] {
- const timings: Timing[] = [];
- for (let i = 0; i < REPETITIONS; i++) {
- const start = new Date().getTime();
- cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS);
- const end = new Date().getTime();
- const timing = end - start;
- timings.push(timing);
- }
- return timings;
-}
-
-function saveCache() {
- // Remove any existing backup
- if (fs.existsSync(ALT_CACHE_PATH)) {
- fs.rmSync(ALT_CACHE_PATH, { recursive: true });
- }
- // copy the current cache to the backup
- if (fs.existsSync(DEFAULT_CACHE_PATH)) {
- fse.copySync(DEFAULT_CACHE_PATH, ALT_CACHE_PATH, { recursive: true });
- } else {
- // make an empty cache
- fs.mkdirSync(ALT_CACHE_PATH, { recursive: true });
- }
-}
-
-function restoreSavedCache() {
- // Remove any existing cache
- if (fs.existsSync(DEFAULT_CACHE_PATH)) {
- fs.rmSync(DEFAULT_CACHE_PATH, { recursive: true });
- }
- // Copy the backed-up cache to the real cache
- fse.copySync(ALT_CACHE_PATH, DEFAULT_CACHE_PATH, { recursive: true });
-}
-
-function cachedBuildWithDelta(): Timing[] {
- // Save existing cache just once, we'll restore from it each time
- saveCache();
-
- // Edit a file in place
- const file = path.join(
- REPO_PATH,
- "packages",
- "crew",
- "important-feature-0",
- "src",
- "lib",
- "important-component-0",
- "important-component-0.tsx"
- );
- const contents = fs.readFileSync(file).toString("utf-8");
- // make a small edit
- const updated = contents.replace("-0!", "-0!!");
- fs.writeFileSync(file, updated);
-
- const timings: Timing[] = [];
- for (let i = 0; i < REPETITIONS; i++) {
- // Make sure we're starting with the cache from before we make the source code edit
- restoreSavedCache();
- const start = new Date().getTime();
- cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS);
- const end = new Date().getTime();
- const timing = end - start;
- timings.push(timing);
- }
- return timings;
-}
-
-function cachedBuildWithDependencyChange(): Timing[] {
- // Save existing cache just once, we'll restore from it each time
- saveCache();
-
- // Edit a dependency
- const file = path.join(REPO_PATH, "apps", "navigation", "package.json");
- const contents = JSON.parse(fs.readFileSync(file).toString("utf-8"));
- contents.dependencies["crew-important-feature-0"] = "*";
- fs.writeFileSync(file, JSON.stringify(contents, null, 2));
-
- const timings: Timing[] = [];
- for (let i = 0; i < REPETITIONS; i++) {
- // Make sure we're starting with the cache from before we made the dependency edit
- restoreSavedCache();
- const start = new Date().getTime();
- cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS);
- const end = new Date().getTime();
- const timing = end - start;
- timings.push(timing);
- }
- return timings;
-}
-
-class Benchmarks {
- private readonly benchmarks: Benchmark[] = [];
- private readonly tbirdEvents: TBirdEvent[] = [];
-
- constructor(
- private readonly benchmarkFile: string,
- private readonly tinybirdFile: string,
- private readonly commitSha: string,
- private readonly commitTimestamp: Date,
- private readonly platform: string
- ) {}
-
- run(name: string, b: () => Timing[]) {
- console.log(name);
- const timings = b();
- const max = Math.max(...timings);
- const min = Math.min(...timings);
- const avg = timings.reduce((a, b) => a + b, 0) / timings.length;
- this.benchmarks.push({
- name,
- value: avg,
- unit: "ms",
- range: String(max - min),
- });
- timings.forEach((t) => {
- this.tbirdEvents.push({
- commitSha: this.commitSha,
- commitTimestamp: this.commitTimestamp,
- platform: this.platform,
- benchmark: name,
- durationMs: t,
- });
- });
- }
-
- flush() {
- console.log(JSON.stringify(this.benchmarks, null, 2));
- fs.writeFileSync(
- this.benchmarkFile,
- JSON.stringify(this.benchmarks, null, 2)
- );
- const stream = ndjson.stringify();
- const fd = fs.openSync(this.tinybirdFile, "w");
- stream.on("data", (line) => {
- fs.writeSync(fd, line);
- });
- this.tbirdEvents.forEach((t) => {
- stream.write(t);
- });
- stream.end();
- fs.closeSync(fd);
- }
-}
-
-cp.execSync(`${TURBO_BIN} --version`, { stdio: "inherit" });
-
-function getCommitDetails(): { commitSha: string; commitTimestamp: Date } {
- const envSha = process.env["GITHUB_SHA"];
- if (envSha === undefined) {
- return {
- commitSha: "unknown sha",
- commitTimestamp: new Date(),
- };
- }
- const buf = cp.execSync(`git show -s --format=%ci ${envSha}`);
- const dateString = String(buf).trim();
- const commitTimestamp = new Date(dateString);
- return {
- commitSha: envSha,
- commitTimestamp,
- };
-}
-
-const { commitSha, commitTimestamp } = getCommitDetails();
-const platform = process.env["RUNNER_OS"] ?? "unknown";
-
-console.log("setup");
-setup();
-const benchmark = new Benchmarks(
- "benchmarks.json",
- "tinybird.ndjson",
- commitSha,
- commitTimestamp,
- platform
-);
-benchmark.run("Clean Build", cleanBuild);
-benchmark.run("Cached Build - No Change", cachedBuild);
-benchmark.run("Cached Build - Code Change", cachedBuildWithDelta);
-benchmark.run(
- "Cached Build - Dependency Change",
- cachedBuildWithDependencyChange
-);
-benchmark.flush();
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2026-01-21 16:44:12 +0000