Data Storage

// Does one relational primary (plus read replicas) carry this workload, or do the numbers force something else?
const DAY = 86_400;

// Rules of thumb for one well-tuned primary on a large instance (not measurements; tune for your hardware).
const ONE_PRIMARY = { writesPerSec: 5_000, dataTB: 5 };

// peak: busiest hour as a multiple of the daily average (3 for human traffic; sensors report at a steady rate).
const scenarios = [
  { name: "shop orders", dau: 2e6, writesPerUser: 0.2, readsPerUser: 20, rowBytes: 1_000, years: 5, peak: 3 },
  { name: "chat messages", dau: 50e6, writesPerUser: 40, readsPerUser: 200, rowBytes: 300, years: 5, peak: 3 },
  { name: "IoT metrics", devices: 1e6, samplesPerDevicePerDay: DAY / 10, readsPerDay: 2e6, rowBytes: 50, years: 1, peak: 1 },
];

const fmt = (n) => (n >= 1e6 ? `${(n / 1e6).toFixed(1)}M` : n >= 1e3 ? `${(n / 1e3).toFixed(1)}k` : n.toFixed(0));

for (const s of scenarios) {
  const writesPerDay = s.dau ? s.dau * s.writesPerUser : s.devices * s.samplesPerDevicePerDay;
  const readsPerDay = s.dau ? s.dau * s.readsPerUser : s.readsPerDay;
  const peakWrites = (writesPerDay / DAY) * s.peak;
  const peakReads = (readsPerDay / DAY) * s.peak;
  const dataTB = (writesPerDay * 365 * s.years * s.rowBytes) / 1e12;

  const problems = [];
  if (peakWrites > ONE_PRIMARY.writesPerSec) problems.push("write rate");
  if (dataTB > ONE_PRIMARY.dataTB) problems.push("data size");
  const verdict = problems.length ? `outgrows one primary (${problems.join(", ")})` : "fits one primary + replicas";

  console.log(`${s.name}: peak ${fmt(peakWrites)} writes/s, ${fmt(peakReads)} reads/s, ${dataTB.toFixed(2)} TB after ${s.years} y`);
  console.log(`  -> ${verdict}`);
}

shop orders: peak 14 writes/s, 1.4k reads/s, 0.73 TB after 5 y
  -> fits one primary + replicas
chat messages: peak 69.4k writes/s, 347.2k reads/s, 1095.00 TB after 5 y
  -> outgrows one primary (write rate, data size)
IoT metrics: peak 100.0k writes/s, 23 reads/s, 157.68 TB after 1 y
  -> outgrows one primary (write rate, data size)

// Rows and index entries touched per day for a feed of posts showing "author name" and "like count", in two data models.
// Seeded PRNG (mulberry32) so the output is the same on every run.
const mulberry32 = (a) => () => {
  a = (a + 0x6d2b79f5) | 0;
  let t = Math.imul(a ^ (a >>> 15), 1 | a);
  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const rand = mulberry32(42);

const POSTS = 10_000, AUTHORS = 2_000, PAGE = 20;
const authorOf = Array.from({ length: POSTS }, () => Math.floor(rand() * AUTHORS));
// Likes are skewed: most posts get a few, a few posts get thousands (Pareto-like tail).
const likes = Array.from({ length: POSTS }, () => Math.floor(5 / Math.pow(rand(), 0.9)));
const postsByAuthor = new Map();
for (const a of authorOf) postsByAuthor.set(a, (postsByAuthor.get(a) ?? 0) + 1);

const day = { pageReads: 500_000, likeWrites: 50_000, authorRenames: 200 };

// Normalised: posts(id, author_id, body), authors(id, name), likes(post_id, user_id).
// A page reads 20 posts + 20 authors + COUNT(*) over each post's likes (one index entry per like).
// Measure it on 1,000 sample pages of 20 random posts.
let sampled = 0;
for (let p = 0; p < 1_000; p++) for (let i = 0; i < PAGE; i++) sampled += likes[Math.floor(rand() * POSTS)];
const avgLikesOnPage = sampled / (1_000 * PAGE);
const normalised = {
  pageRead: PAGE + PAGE + PAGE * avgLikesOnPage,
  likeWrite: 1,
  rename: 1,
};

// Denormalised: posts also store author_name and like_count; a like updates the counter too,
// and a rename rewrites every post by that author.
const avgPostsPerAuthor = POSTS / postsByAuthor.size;
const denormalised = { pageRead: PAGE, likeWrite: 2, rename: 1 + avgPostsPerAuthor };

for (const [name, m] of Object.entries({ normalised, denormalised })) {
  const total = day.pageReads * m.pageRead + day.likeWrites * m.likeWrite + day.authorRenames * m.rename;
  console.log(
    `${name.padEnd(12)} page read ${m.pageRead.toFixed(0).padStart(6)} entries | like ${m.likeWrite} | ` +
      `rename ${m.rename.toFixed(1)} | per day ${(total / 1e6).toFixed(1)}M`,
  );
}
console.log(`avg likes on a feed post ${avgLikesOnPage.toFixed(0)}, avg posts per author ${avgPostsPerAuthor.toFixed(1)}`);

normalised   page read    552 entries | like 1 | rename 1.0 | per day 276.1M
denormalised page read     20 entries | like 2 | rename 6.0 | per day 10.1M
avg likes on a feed post 26, avg posts per author 5.0

// Asynchronous leader-follower replication: how often does a user fail to see their own write?
// Seeded PRNG (mulberry32); times are simulated milliseconds, not measured.
const mulberry32 = (a) => () => {
  a = (a + 0x6d2b79f5) | 0;
  let t = Math.imul(a ^ (a >>> 15), 1 | a);
  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const rand = mulberry32(7);

// Replication lag per write: usually 5-50 ms, but 2% of the time a follower is behind by 0.5-5 s
// (a big transaction, a vacuum, a backup, a network blip).
const lag = () => (rand() < 0.02 ? 500 + rand() * 4_500 : 5 + rand() * 45);

// One sample = a user saves, then reads back after a think time of 50 ms (page redirect) to 3 s,
// from the follower the load balancer picks. The same samples are replayed under each routing policy.
const samples = Array.from({ length: 100_000 }, () => ({ lag: lag(), readAt: 50 + rand() * 2_950 }));

const policies = {
  "any follower": () => "follower",
  "leader for 1 s after own write": (s) => (s.readAt < 1_000 ? "leader" : "follower"),
  // Compare the follower's applied log position with the position of the user's last write
  // (MySQL GTID, Postgres LSN, MongoDB operationTime); a session token carries the latter.
  "follower only if caught up": (s) => (s.lag <= s.readAt ? "follower" : "leader"),
};

for (const [name, route] of Object.entries(policies)) {
  let stale = 0, leaderReads = 0;
  for (const s of samples) {
    if (route(s) === "leader") leaderReads++;
    else if (s.lag > s.readAt) stale++; // the follower hasn't applied the write yet
  }
  const pct = (n) => `${((100 * n) / samples.length).toFixed(2)}%`;
  console.log(`${name.padEnd(31)} stale reads ${pct(stale).padStart(6)}   reads on leader ${pct(leaderReads).padStart(6)}`);
}

any follower                    stale reads  1.51%   reads on leader  0.00%
leader for 1 s after own write  stale reads  0.89%   reads on leader 32.29%
follower only if caught up      stale reads  0.00%   reads on leader  1.51%

// Range vs hash sharding on realistic keys: how much of the load lands on the busiest of 8 shards?
// Seeded PRNG (mulberry32), so the output is the same on every run.
const mulberry32 = (a) => () => {
  a = (a + 0x6d2b79f5) | 0;
  let t = Math.imul(a ^ (a >>> 15), 1 | a);
  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const rand = mulberry32(2026);
const fnv1a = (s) => { let h = 0x811c9dc5; for (const c of s) h = Math.imul(h ^ c.charCodeAt(0), 0x01000193); return h >>> 0; };

const SHARDS = 8, OPS = 200_000;
const hashShard = (key) => fnv1a(String(key)) % SHARDS;
const rangeShard = (bounds) => (key) => { let i = 0; while (i < SHARDS - 1 && key >= bounds[i]) i++; return i; };

// Workload 1: events keyed by timestamp. The table holds 80 days; ranges were split evenly
// over those 80 days (10 days per shard). Today's writes all carry today's time.
const DAY = 86_400_000, start = Date.UTC(2026, 7, 1);
const timeBounds = Array.from({ length: SHARDS - 1 }, (_, i) => start + (i + 1) * 10 * DAY);
const eventKeys = Array.from({ length: OPS }, () => start + 79 * DAY + Math.floor(rand() * DAY));

// Workload 2: requests keyed by user id. 1M users with Zipf-like activity (a few users are very busy,
// most are quiet), unrelated to id order, plus one celebrity (id 777) who gets 5% of all traffic.
// Ranges split the id space evenly (125k ids per shard).
const USERS = 1_000_000;
const userBounds = Array.from({ length: SHARDS - 1 }, (_, i) => (i + 1) * (USERS / SHARDS));
const rank = () => Math.floor(Math.pow(USERS, rand())); // log-uniform rank: rank 1 is the busiest
const idOf = (r) => (r * 387_420_489) % USERS; // scatter ranks over ids (3^18 is coprime with 1M)
const userKeys = Array.from({ length: OPS }, () => (rand() < 0.05 ? 777 : idOf(rank())));

function report(label, keys, shardOf) {
  const load = new Array(SHARDS).fill(0);
  for (const k of keys) load[shardOf(k)]++;
  const pct = load.map((n) => Math.round((100 * n) / keys.length));
  const maxOverMean = Math.max(...load) / (keys.length / SHARDS);
  console.log(`${label.padEnd(20)} ${pct.map((p) => String(p).padStart(4)).join("")}   busiest ${maxOverMean.toFixed(2)}x mean`);
}

console.log(`${"".padEnd(20)} ${Array.from({ length: SHARDS }, (_, i) => `s${i}`.padStart(4)).join("")}   (% of operations)`);
report("timestamps, range", eventKeys, rangeShard(timeBounds));
report("timestamps, hash", eventKeys, hashShard);
report("user ids, range", userKeys, rangeShard(userBounds));
report("user ids, hash", userKeys, hashShard);
const counts = new Map();
for (const k of userKeys) counts.set(k, (counts.get(k) ?? 0) + 1);
const top = [...counts].sort((a, b) => b[1] - a[1]).slice(0, 4);
console.log("hottest user ids:", top.map(([k, n]) => `${k} ${((100 * n) / OPS).toFixed(1)}% -> s${hashShard(k)}`).join(", "));

                       s0  s1  s2  s3  s4  s5  s6  s7   (% of operations)
timestamps, range       0   0   0   0   0   0   0 100   busiest 8.00x mean
timestamps, hash       12  12  12  13  13  13  13  13   busiest 1.01x mean
user ids, range        16  10  13  15  11  12  13  11   busiest 1.28x mean
user ids, hash         12  13  12  12  21  13   8  10   busiest 1.67x mean
hottest user ids: 777 5.0% -> s4, 420489 4.8% -> s4, 840978 2.8% -> s1, 261467 1.9% -> s3

// A celebrity's like counter takes 30% of 10,000 writes/s on a table hashed over 16 partitions.
// Each partition can absorb 1,000 writes/s (DynamoDB's documented per-partition write limit).
// Fix: split the hot key into N sub-keys ("salting"); writers pick one at random, readers sum all N.
// Seeded PRNG (mulberry32).
const mulberry32 = (a) => () => {
  a = (a + 0x6d2b79f5) | 0;
  let t = Math.imul(a ^ (a >>> 15), 1 | a);
  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const fnv1a = (s) => { let h = 0x811c9dc5; for (const c of s) h = Math.imul(h ^ c.charCodeAt(0), 0x01000193); return h >>> 0; };

const PARTITIONS = 16, WRITES_PER_SEC = 10_000, HOT_SHARE = 0.3, LIMIT = 1_000;
const partitionOf = (key) => fnv1a(key) % PARTITIONS;

function oneSecond(salt) {
  const rand = mulberry32(99); // same traffic for every run
  const pick = mulberry32(5); // separate stream for choosing a sub-key, so the traffic itself doesn't change
  const load = new Array(PARTITIONS).fill(0);
  const counts = new Map();
  for (let i = 0; i < WRITES_PER_SEC; i++) {
    let key = rand() < HOT_SHARE ? "likes:post:celebrity" : `likes:post:${Math.floor(rand() * 50_000)}`;
    if (key.endsWith("celebrity") && salt > 1) key += `#${Math.floor(pick() * salt)}`; // pick a sub-key
    load[partitionOf(key)]++;
    counts.set(key, (counts.get(key) ?? 0) + 1);
  }
  // Reading the celebrity's total now costs `salt` reads, one per sub-key.
  const subKeys = salt > 1 ? Array.from({ length: salt }, (_, i) => `likes:post:celebrity#${i}`) : ["likes:post:celebrity"];
  const total = subKeys.reduce((sum, k) => sum + (counts.get(k) ?? 0), 0);
  const throttled = load.reduce((sum, n) => sum + Math.max(0, n - LIMIT), 0);
  const used = new Set(subKeys.map(partitionOf)).size;
  console.log(
    `salt ${String(salt).padStart(2)}: busiest partition ${String(Math.max(...load)).padStart(5)} w/s, ` +
      `throttled ${String(throttled).padStart(4)} w/s, hot key on ${used} partition(s), read = ${subKeys.length} gets -> ${total}`,
  );
}

for (const salt of [1, 4, 16]) oneSecond(salt);

salt  1: busiest partition  3463 w/s, throttled 2463 w/s, hot key on 1 partition(s), read = 1 gets -> 3054
salt  4: busiest partition  1239 w/s, throttled  804 w/s, hot key on 4 partition(s), read = 4 gets -> 3054
salt 16: busiest partition   822 w/s, throttled    0 w/s, hot key on 10 partition(s), read = 16 gets -> 3054

// Query by a non-key attribute ("orders with status = 'disputed'") on N shards.
// Local index: every shard indexes its own rows, so the query asks all N shards and waits for the slowest.
// Global index: one index partition knows the matching ids; then fetch rows from the few shards that hold them.
// Simulated latencies from a seeded PRNG (mulberry32), not measurements.
const mulberry32 = (a) => () => {
  a = (a + 0x6d2b79f5) | 0;
  let t = Math.imul(a ^ (a >>> 15), 1 | a);
  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const rand = mulberry32(11);

// One shard call: 2-8 ms usually; 1% of calls hit a GC pause, a compaction or a cold cache: 50-150 ms.
const call = () => (rand() < 0.01 ? 50 + rand() * 100 : 2 + rand() * 6);
const QUERIES = 20_000;
const MATCH_SHARDS = 3; // the matching rows live on 3 shards

const pct = (arr, p) => [...arr].sort((a, b) => a - b)[Math.floor((p / 100) * (arr.length - 1))];

const ms = (arr) => [50, 90, 99].map((p) => pct(arr, p).toFixed(0).padStart(4)).join("");
console.log("       |   local index (scatter-gather)  |   global index (lookup + fetch)");
console.log("shards | calls/query  p50 p90 p99 (ms)  | calls/query  p50 p90 p99 (ms)");
for (const N of [4, 16, 64]) {
  const local = [], global = [];
  for (let q = 0; q < QUERIES; q++) {
    local.push(Math.max(...Array.from({ length: N }, call))); // scatter to all N, gather when the last replies
    const fetch = Math.max(...Array.from({ length: Math.min(N, MATCH_SHARDS) }, call));
    global.push(call() + fetch); // index lookup, then parallel fetches
  }
  console.log(`${String(N).padStart(6)} | ${String(N).padStart(11)} ${ms(local)}       | ${String(1 + Math.min(N, MATCH_SHARDS)).padStart(11)} ${ms(global)}`);
}
const slow = (N) => (100 * (1 - 0.99 ** N)).toFixed(0);
console.log(`chance at least one of N calls is slow: N=4 ${slow(4)}%, N=16 ${slow(16)}%, N=64 ${slow(64)}%`);

       |   local index (scatter-gather)  |   global index (lookup + fetch)
shards | calls/query  p50 p90 p99 (ms)  | calls/query  p50 p90 p99 (ms)
     4 |           4    7   8 124       |           4   12  15 130
    16 |          16    8  84 144       |           4   12  15 131
    64 |          64    8 134 148       |           4   12  15 133
chance at least one of N calls is slow: N=4 4%, N=16 15%, N=64 47%

// An S3 presigned URL (AWS Signature Version 4, query-string form), with node:crypto only.
// Inputs are the worked example in AWS's S3 API docs ("Authenticating Requests: Using Query Parameters"),
// including AWS's published example credentials, so the result can be checked against the docs.
import { createHash, createHmac } from "node:crypto";

const sha256 = (s) => createHash("sha256").update(s).digest("hex");
const hmac = (key, s) => createHmac("sha256", key).update(s).digest();
const enc = (s) => encodeURIComponent(s).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);

function presign({ method, bucket, key, region, accessKeyId, secretAccessKey, amzDate, expiresSec }) {
  const host = `${bucket}.s3.amazonaws.com`;
  const date = amzDate.slice(0, 8);
  const scope = `${date}/${region}/s3/aws4_request`;
  const params = {
    "X-Amz-Algorithm": "AWS4-HMAC-SHA256",
    "X-Amz-Credential": `${accessKeyId}/${scope}`,
    "X-Amz-Date": amzDate,
    "X-Amz-Expires": String(expiresSec),
    "X-Amz-SignedHeaders": "host",
  };
  const query = Object.keys(params).sort().map((k) => `${enc(k)}=${enc(params[k])}`).join("&");
  // 1. Canonical request: exactly what is being allowed (method, object path, query, signed headers).
  const canonical = [method, `/${key.split("/").map(enc).join("/")}`, query, `host:${host}`, "", "host", "UNSIGNED-PAYLOAD"].join("\n");
  // 2. String to sign: algorithm, time, scope and a hash of the canonical request.
  const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256(canonical)].join("\n");
  // 3. Signing key: the secret is never sent; it's folded through date, region and service.
  let k = hmac(`AWS4${secretAccessKey}`, date);
  for (const part of [region, "s3", "aws4_request"]) k = hmac(k, part);
  const signature = createHmac("sha256", k).update(stringToSign).digest("hex");
  return { url: `https://${host}/${key}?${query}&X-Amz-Signature=${signature}`, stringToSign, signature };
}

const { url, stringToSign, signature } = presign({
  method: "GET",
  bucket: "examplebucket",
  key: "test.txt",
  region: "us-east-1",
  accessKeyId: "AKIAIOSFODNN7EXAMPLE",
  secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
  amzDate: "20130524T000000Z",
  expiresSec: 86_400,
});
console.log(stringToSign);
console.log(url.replace("?", "\n  ?").replaceAll("&", "\n  &"));
console.log("matches the AWS docs:", signature === "aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404");

AWS4-HMAC-SHA256
20130524T000000Z
20130524/us-east-1/s3/aws4_request
3bfa292879f6447bbcda7001decf97f4a54dc650c8942174ae0a9121cf58ad04
https://examplebucket.s3.amazonaws.com/test.txt
  ?X-Amz-Algorithm=AWS4-HMAC-SHA256
  &X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request
  &X-Amz-Date=20130524T000000Z
  &X-Amz-Expires=86400
  &X-Amz-SignedHeaders=host
  &X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404
matches the AWS docs: true

// A tiny search engine: tokenise, build postings lists, rank with BM25 (Lucene/Elasticsearch defaults k1=1.2, b=0.75).
const docs = {
  1: "Wireless noise cancelling headphones with 30 hour battery",
  2: "Wired headphones for studio monitoring",
  3: "Noise cancelling earbuds, wireless charging case",
  4: "Bluetooth speaker with deep bass and 12 hour battery",
  5: "Over-ear headphones, wireless, foldable, with noise cancelling and a travel case for flights",
  6: "Phone case with wireless charging support",
};

const STOP = new Set(["with", "for", "and", "a", "the", "of"]);
// Analyzer: lowercase, split on anything that isn't a letter or digit, drop stopwords, strip a plural "s" (not "ss").
const stem = (t) => (t.length > 3 && t.endsWith("s") && !t.endsWith("ss") ? t.slice(0, -1) : t);
const analyze = (text) => text.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t && !STOP.has(t)).map(stem);

// Inverted index: term -> postings list of [docId, term frequency], plus each doc's length in terms.
const index = new Map();
const docLen = new Map();
for (const [id, text] of Object.entries(docs)) {
  const terms = analyze(text);
  docLen.set(id, terms.length);
  const tf = new Map();
  for (const t of terms) tf.set(t, (tf.get(t) ?? 0) + 1);
  for (const [t, n] of tf) (index.get(t) ?? index.set(t, []).get(t)).push([id, n]);
}
const N = docLen.size;
const avgLen = [...docLen.values()].reduce((a, b) => a + b, 0) / N;

function search(query, { mode = "or", k1 = 1.2, b = 0.75 } = {}) {
  const qTerms = [...new Set(analyze(query))];
  const scores = new Map(), matched = new Map();
  for (const t of qTerms) {
    const postings = index.get(t) ?? [];
    const idf = Math.log(1 + (N - postings.length + 0.5) / (postings.length + 0.5)); // rare terms weigh more
    for (const [id, tf] of postings) {
      const norm = tf + k1 * (1 - b + (b * docLen.get(id)) / avgLen); // long docs need more repeats to score
      scores.set(id, (scores.get(id) ?? 0) + (idf * tf * (k1 + 1)) / norm);
      matched.set(id, (matched.get(id) ?? 0) + 1);
    }
  }
  return [...scores]
    .filter(([id]) => mode === "or" || matched.get(id) === qTerms.length)
    .sort((x, y) => y[1] - x[1])
    .map(([id, s]) => `doc ${id} ${s.toFixed(3)}`);
}

console.log("postings 'wireless':", JSON.stringify(index.get("wireless")));
console.log("postings 'headphone':", JSON.stringify(index.get("headphone")));
console.log("analyze:", analyze("Noise-Cancelling Headphones"));
console.log("OR  'wireless noise cancelling headphones':", search("wireless noise cancelling headphones").join(", "));
console.log("AND 'wireless noise cancelling headphones':", search("wireless noise cancelling headphones", { mode: "and" }).join(", "));
console.log("OR  'battery case':", search("battery case").join(", "));

postings 'wireless': [["1",1],["3",1],["5",1],["6",1]]
postings 'headphone': [["1",1],["2",1],["5",1]]
analyze: [ 'noise', 'cancelling', 'headphone' ]
OR  'wireless noise cancelling headphones': doc 1 2.444, doc 5 2.066, doc 3 1.888, doc 2 0.823, doc 6 0.488
AND 'wireless noise cancelling headphones': doc 1 2.444, doc 5 2.066
OR  'battery case': doc 1 0.998, doc 4 0.998, doc 6 0.765, doc 3 0.716, doc 5 0.568

// Downsampling a latency metric: raw request latencies -> 1-minute rollups -> 1-hour rollup.
// Seeded PRNG (mulberry32). Traffic is uneven: a busy minute has 50x the requests of a quiet one.
const mulberry32 = (a) => () => {
  a = (a + 0x6d2b79f5) | 0;
  let t = Math.imul(a ^ (a >>> 15), 1 | a);
  t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
const rand = mulberry32(3);

// 60 minutes. Minutes 20-29 are a traffic spike (5,000 requests each, slow: 120-400 ms);
// the other minutes are quiet (100 requests each, fast: 20-60 ms).
const raw = [];
for (let m = 0; m < 60; m++) {
  const spike = m >= 20 && m < 30;
  for (let i = 0; i < (spike ? 5_000 : 100); i++) raw.push({ m, ms: spike ? 120 + rand() * 280 : 20 + rand() * 40 });
}

const p99 = (xs) => [...xs].sort((a, b) => a - b)[Math.floor(0.99 * (xs.length - 1))];

// 1-minute rollups keep count, sum, min, max (all mergeable) and that minute's p99 (not mergeable).
const minutes = Array.from({ length: 60 }, (_, m) => {
  const xs = raw.filter((r) => r.m === m).map((r) => r.ms);
  return { count: xs.length, sum: xs.reduce((a, b) => a + b, 0), min: Math.min(...xs), max: Math.max(...xs), p99: p99(xs), avg: 0 };
});
for (const r of minutes) r.avg = r.sum / r.count;

const all = raw.map((r) => r.ms);
const trueAvg = all.reduce((a, b) => a + b, 0) / all.length;
const avgOfAvgs = minutes.reduce((a, r) => a + r.avg, 0) / 60;
const fromSumCount = minutes.reduce((a, r) => a + r.sum, 0) / minutes.reduce((a, r) => a + r.count, 0);
const avgOfP99s = minutes.reduce((a, r) => a + r.p99, 0) / 60;
const maxOfP99s = Math.max(...minutes.map((r) => r.p99));

console.log(`requests in the hour: ${all.length}`);
console.log(`hour average   true ${trueAvg.toFixed(1)} ms | sum/count of rollups ${fromSumCount.toFixed(1)} ms | avg of minute avgs ${avgOfAvgs.toFixed(1)} ms`);
console.log(`hour p99       true ${p99(all).toFixed(1)} ms | avg of minute p99s ${avgOfP99s.toFixed(1)} ms | max of minute p99s ${maxOfP99s.toFixed(1)} ms`);
console.log(`hour min/max   ${Math.min(...minutes.map((r) => r.min)).toFixed(1)} / ${Math.max(...minutes.map((r) => r.max)).toFixed(1)} ms (exact from rollups)`);