4 files changed, 298 insertions, 0 deletions

diff --git a/benchmark/.gitignore b/benchmark/.gitignore
new file mode 100644
index 0000000..956951a
--- /dev/null
+++ b/benchmark/.gitignore
@@ -0,0 +1,3 @@
+large-monorepo
+benchmarks.json
+tinybird.ndjson
diff --git a/benchmark/README.md b/benchmark/README.md
new file mode 100644
index 0000000..2f13f23
--- /dev/null
+++ b/benchmark/README.md
@@ -0,0 +1,6 @@
+# Turborepo Benchmarking
+
+To run benchmarks for turborepo
+
+1. From `../cli/` run `make turbo-prod` to build turbo
+2. From this directory `pnpm run benchmark`
diff --git a/benchmark/package.json b/benchmark/package.json
new file mode 100644
index 0000000..9a92660
--- /dev/null
+++ b/benchmark/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "benchmark",
+  "version": "1.0.0",
+  "dependencies": {
+    "esbuild": "^0.15.0",
+    "esbuild-register": "^3.3.2",
+    "fs-extra": "^10.0.0",
+    "ndjson": "^2.0.0"
+  },
+  "scripts": {
+    "benchmark": "node -r esbuild-register src/index.ts"
+  },
+  "devDependencies": {
+    "@types/node": "^16.11.49"
+  }
+}
diff --git a/benchmark/src/index.ts b/benchmark/src/index.ts
new file mode 100644
index 0000000..117d6b1
--- /dev/null
+++ b/benchmark/src/index.ts
@@ -0,0 +1,273 @@
+import cp from "child_process";
+import fs from "fs";
+import fse from "fs-extra";
+import path from "path";
+import ndjson from "ndjson";
+
+const REPO_ROOT = "large-monorepo";
+const REPO_ORIGIN = "https://github.com/gsoltis/large-monorepo.git";
+const REPO_PATH = path.join(process.cwd(), REPO_ROOT);
+const REPETITIONS = 5;
+
+const DEFAULT_EXEC_OPTS = { stdio: "ignore" as const, cwd: REPO_PATH };
+const TURBO_BIN = path.resolve(path.join("..", "target", "release", "turbo"));
+const DEFAULT_CACHE_PATH = path.join(
+  REPO_PATH,
+  "node_modules",
+  ".cache",
+  "turbo"
+);
+const ALT_CACHE_PATH = path.join(
+  REPO_PATH,
+  "node_modules",
+  ".cache",
+  "turbo-benchmark"
+);
+
+type Timing = number;
+
+type Benchmark = {
+  name: string;
+  unit: string;
+  value: number;
+  range?: string;
+  extra?: string;
+};
+
+type TBirdEvent = {
+  commitSha: string;
+  commitTimestamp: Date;
+  platform: string;
+  benchmark: string;
+  durationMs: number;
+};
+
+function setup(): void {
+  // Clone repo if it doesn't exist, run clean
+  if (fs.existsSync(REPO_ROOT)) {
+    // reset the repo, remove all changed or untracked files
+    cp.execSync(
+      `cd ${REPO_ROOT} && git reset --hard HEAD && git clean -f -d -X`,
+      {
+        stdio: "inherit",
+      }
+    );
+  } else {
+    cp.execSync(`git clone ${REPO_ORIGIN}`, { stdio: "ignore" });
+  }
+
+  // Run install so we aren't benchmarking node_modules ...
+
+  cp.execSync("yarn install", DEFAULT_EXEC_OPTS);
+}
+
+function cleanTurboCache(): void {
+  if (fs.existsSync(DEFAULT_CACHE_PATH)) {
+    console.log("clearing cache");
+    fs.rmSync(DEFAULT_CACHE_PATH, { recursive: true });
+  }
+}
+
+function cleanBuild(): Timing[] {
+  const timings: Timing[] = [];
+  const isLocal = process.argv[process.argv.length - 1] == "--local";
+  // We aren't really benchmarking this one, it OOMs if run in full parallel
+  // on GH actions
+  const repetitions = isLocal ? REPETITIONS : 1;
+  const concurrency = isLocal ? "" : " --concurrency=1";
+  for (let i = 0; i < repetitions; i++) {
+    // clean first, we'll leave the cache in place for subsequent builds
+    cleanTurboCache();
+    const start = new Date().getTime();
+    cp.execSync(`${TURBO_BIN} run build${concurrency}`, DEFAULT_EXEC_OPTS);
+    const end = new Date().getTime();
+    const timing = end - start;
+    timings.push(timing);
+  }
+  return timings;
+}
+
+function cachedBuild(): Timing[] {
+  const timings: Timing[] = [];
+  for (let i = 0; i < REPETITIONS; i++) {
+    const start = new Date().getTime();
+    cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS);
+    const end = new Date().getTime();
+    const timing = end - start;
+    timings.push(timing);
+  }
+  return timings;
+}
+
+function saveCache() {
+  // Remove any existing backup
+  if (fs.existsSync(ALT_CACHE_PATH)) {
+    fs.rmSync(ALT_CACHE_PATH, { recursive: true });
+  }
+  // copy the current cache to the backup
+  if (fs.existsSync(DEFAULT_CACHE_PATH)) {
+    fse.copySync(DEFAULT_CACHE_PATH, ALT_CACHE_PATH, { recursive: true });
+  } else {
+    // make an empty cache
+    fs.mkdirSync(ALT_CACHE_PATH, { recursive: true });
+  }
+}
+
+function restoreSavedCache() {
+  // Remove any existing cache
+  if (fs.existsSync(DEFAULT_CACHE_PATH)) {
+    fs.rmSync(DEFAULT_CACHE_PATH, { recursive: true });
+  }
+  // Copy the backed-up cache to the real cache
+  fse.copySync(ALT_CACHE_PATH, DEFAULT_CACHE_PATH, { recursive: true });
+}
+
+function cachedBuildWithDelta(): Timing[] {
+  // Save existing cache just once, we'll restore from it each time
+  saveCache();
+
+  // Edit a file in place
+  const file = path.join(
+    REPO_PATH,
+    "packages",
+    "crew",
+    "important-feature-0",
+    "src",
+    "lib",
+    "important-component-0",
+    "important-component-0.tsx"
+  );
+  const contents = fs.readFileSync(file).toString("utf-8");
+  // make a small edit
+  const updated = contents.replace("-0!", "-0!!");
+  fs.writeFileSync(file, updated);
+
+  const timings: Timing[] = [];
+  for (let i = 0; i < REPETITIONS; i++) {
+    // Make sure we're starting with the cache from before we make the source code edit
+    restoreSavedCache();
+    const start = new Date().getTime();
+    cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS);
+    const end = new Date().getTime();
+    const timing = end - start;
+    timings.push(timing);
+  }
+  return timings;
+}
+
+function cachedBuildWithDependencyChange(): Timing[] {
+  // Save existing cache just once, we'll restore from it each time
+  saveCache();
+
+  // Edit a dependency
+  const file = path.join(REPO_PATH, "apps", "navigation", "package.json");
+  const contents = JSON.parse(fs.readFileSync(file).toString("utf-8"));
+  contents.dependencies["crew-important-feature-0"] = "*";
+  fs.writeFileSync(file, JSON.stringify(contents, null, 2));
+
+  const timings: Timing[] = [];
+  for (let i = 0; i < REPETITIONS; i++) {
+    // Make sure we're starting with the cache from before we made the dependency edit
+    restoreSavedCache();
+    const start = new Date().getTime();
+    cp.execSync(`${TURBO_BIN} run build`, DEFAULT_EXEC_OPTS);
+    const end = new Date().getTime();
+    const timing = end - start;
+    timings.push(timing);
+  }
+  return timings;
+}
+
+class Benchmarks {
+  private readonly benchmarks: Benchmark[] = [];
+  private readonly tbirdEvents: TBirdEvent[] = [];
+
+  constructor(
+    private readonly benchmarkFile: string,
+    private readonly tinybirdFile: string,
+    private readonly commitSha: string,
+    private readonly commitTimestamp: Date,
+    private readonly platform: string
+  ) {}
+
+  run(name: string, b: () => Timing[]) {
+    console.log(name);
+    const timings = b();
+    const max = Math.max(...timings);
+    const min = Math.min(...timings);
+    const avg = timings.reduce((a, b) => a + b, 0) / timings.length;
+    this.benchmarks.push({
+      name,
+      value: avg,
+      unit: "ms",
+      range: String(max - min),
+    });
+    timings.forEach((t) => {
+      this.tbirdEvents.push({
+        commitSha: this.commitSha,
+        commitTimestamp: this.commitTimestamp,
+        platform: this.platform,
+        benchmark: name,
+        durationMs: t,
+      });
+    });
+  }
+
+  flush() {
+    console.log(JSON.stringify(this.benchmarks, null, 2));
+    fs.writeFileSync(
+      this.benchmarkFile,
+      JSON.stringify(this.benchmarks, null, 2)
+    );
+    const stream = ndjson.stringify();
+    const fd = fs.openSync(this.tinybirdFile, "w");
+    stream.on("data", (line) => {
+      fs.writeSync(fd, line);
+    });
+    this.tbirdEvents.forEach((t) => {
+      stream.write(t);
+    });
+    stream.end();
+    fs.closeSync(fd);
+  }
+}
+
+cp.execSync(`${TURBO_BIN} --version`, { stdio: "inherit" });
+
+function getCommitDetails(): { commitSha: string; commitTimestamp: Date } {
+  const envSha = process.env["GITHUB_SHA"];
+  if (envSha === undefined) {
+    return {
+      commitSha: "unknown sha",
+      commitTimestamp: new Date(),
+    };
+  }
+  const buf = cp.execSync(`git show -s --format=%ci ${envSha}`);
+  const dateString = String(buf).trim();
+  const commitTimestamp = new Date(dateString);
+  return {
+    commitSha: envSha,
+    commitTimestamp,
+  };
+}
+
+const { commitSha, commitTimestamp } = getCommitDetails();
+const platform = process.env["RUNNER_OS"] ?? "unknown";
+
+console.log("setup");
+setup();
+const benchmark = new Benchmarks(
+  "benchmarks.json",
+  "tinybird.ndjson",
+  commitSha,
+  commitTimestamp,
+  platform
+);
+benchmark.run("Clean Build", cleanBuild);
+benchmark.run("Cached Build - No Change", cachedBuild);
+benchmark.run("Cached Build - Code Change", cachedBuildWithDelta);
+benchmark.run(
+  "Cached Build - Dependency Change",
+  cachedBuildWithDependencyChange
+);
+benchmark.flush();