Remove generation profile tiering; use explicit runs count

2026-04-30 13:56:39 -07:00
parent 6295c476dc
commit 099576f12a
13 changed files with 41 additions and 58 deletions
@@ -96,7 +96,7 @@ See [docs/getting-started.md](docs/getting-started.md) for the full walkthrough.
 ## Trust and Safety

 - **Deterministic replay**: Every failure includes a seed and a one-command replay.
- **Generation profile aliases**: Control test budget with `--generation-profile quick|standard|deep`.
+- **Explicit test budget**: Control how many tests run with `runs: 10` in your preset.
 - **CI-safe default path**: `verify` is deterministic and safe for CI pipelines.
 - **Machine-readable output**: `--format json-summary` and `--format ndjson-summary` for CI dashboards.
 - **Production-safe observe path**: `observe` is non-blocking by default. Blocking behavior requires explicit break-glass policy.
@@ -123,7 +123,7 @@ app.post('/users', {
 })

 await app.ready()
-const suite = await app.apophis.contract({ depth: 'standard' })
+const suite = await app.apophis.contract({ runs: 50 })
 ```

 ## API Surface
@@ -8,7 +8,7 @@ Chaos testing applies the invariant-driven verification approach from [Invariant

 ```javascript
 const result = await fastify.apophis.contract({
-  depth: 'standard',
+  runs: 50,
  chaos: {
    probability: 0.1,  // 10% of requests get chaos
    delay: { probability: 1, minMs: 100, maxMs: 500 },
@@ -10,7 +10,6 @@ Every command accepts these flags:
 |---|---|---|
 | `--config <path>` | Config file path | Auto-detect |
 | `--profile <name>` | Profile name from config | First profile |
-| `--generation-profile <name>` | Generation budget profile (built-in: quick, standard, deep) | Depth-derived |
 | `--cwd <path>` | Working directory override | `process.cwd()` |
 | `--format <mode>` | Output format: `human`, `json`, `ndjson`, `json-summary`, `ndjson-summary` | `human` |
 | `--color <mode>` | Color mode: `auto`, `always`, `never` | `auto` |
@@ -62,7 +61,6 @@ apophis verify --profile quick --routes "POST /users"
 | Flag | Description |
 |---|---|
 | `--profile <name>` | Profile name from config |
-| `--generation-profile <name>` | Override generation budget for this run |
 | `--routes <filter>` | Route filter pattern (comma-separated, supports wildcards) |
 | `--seed <number>` | Deterministic seed (generated and printed if omitted) |
 | `--changed` | Filter to git-modified routes only |
@@ -121,7 +119,6 @@ apophis qualify --profile oauth-nightly --seed 42
 | Flag | Description |
 |---|---|
 | `--profile <name>` | Profile name from config |
-| `--generation-profile <name>` | Override generation budget for this run |
 | `--seed <number>` | Deterministic seed (generated and printed if omitted) |

 **Examples:**
@@ -129,18 +126,6 @@ apophis qualify --profile oauth-nightly --seed 42
 ```bash
 apophis qualify --profile oauth-nightly --seed 42
 apophis qualify --profile lifecycle-deep
-apophis qualify --profile oauth-nightly --generation-profile quick
-```
-
-You can define aliases in config:
-
-```js
-export default {
-  generationProfiles: {
-    pr: 'quick',
-    nightly: { base: 'thorough' },
-  },
-}
 ```

 ### `apophis replay`
@@ -153,11 +153,11 @@ fastify.delete('/users/:id', {
 await fastify.ready()

 // Run contract tests (all non-utility routes, property-based)
-const result = await fastify.apophis.contract({ depth: 'standard' })
+const result = await fastify.apophis.contract({ runs: 50 })
 console.log('Contract tests:', result.summary)

 // Run stateful tests (constructor→mutator→destructor sequences)
-const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 })
+const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
 console.log('Stateful tests:', stateful.summary)

 // Validate a single route
@@ -22,5 +22,5 @@ fastify.get('/health', {
 await fastify.ready()

 // Run contract tests
-const result = await fastify.apophis.contract({ depth: 'quick' })
+const result = await fastify.apophis.contract({ runs: 10 })
 console.log(result.summary)
@@ -86,7 +86,6 @@ export default {
  presets: {
    'llm-safe': {
      name: 'llm-safe',
-      depth: 'quick',
      timeout: 3000,
      parallel: false,
      chaos: false,
@@ -135,7 +135,6 @@ export default {
  presets: {
    'platform-observe': {
      name: 'platform-observe',
-      depth: 'standard',
      timeout: 10000,
      parallel: true,
      chaos: false,
@@ -23,8 +23,8 @@ BENCH_RUNS=12 BENCH_WARMUP=3 npm run benchmark:cli
 # Increase inner-loop work for micro-benchmarks
 BENCH_INNER_ITERS=5000 npm run benchmark:hot

-# Benchmark generation profile matrix
-BENCH_GENERATION_PROFILES=quick,standard,thorough npm run benchmark:all
+# Benchmark with varying test counts
+BENCH_RUNS=10,50,200 npm run benchmark:all
 ```

 ## Capture CPU Profile for Qualify
@@ -41,10 +41,10 @@ This writes Chrome-compatible CPU profiles to `.profiles/qualify.cpuprofile` and
 - CLI benchmark uses spawned `node dist/cli/index.js` commands so startup costs are included.
 - Hot path benchmark runs in-process for lower-noise function-level comparisons.
 - Use fixed `--seed` for qualify benchmarks to keep runs deterministic.
- Generation now adapts to depth: `quick` favors bounded payload generation speed, `thorough` keeps broader generation.
+- Schema generation uses fixed defaults (string≤128, array≤10) regardless of run count.

-You can override generation per run:
+You can override runs per preset:

 ```bash
-apophis qualify --profile oauth-nightly --generation-profile quick --seed 42
+apophis qualify --profile oauth-nightly --seed 42
 ```
@@ -97,7 +97,7 @@ APOPHIS tracks created resources and runs cleanup after test completion.
 Run stateful tests via the API:

 ```javascript
-const stateful = await fastify.apophis.stateful({ depth: 'standard', seed: 42 })
+const stateful = await fastify.apophis.stateful({ runs: 50, seed: 42 })
 console.log('Stateful tests:', stateful.summary)
 ```

@@ -203,7 +203,7 @@ export default {
  presets: {
    'protocol-lab': {
      name: 'protocol-lab',
-      depth: 'deep',
+    runs: 200,
      timeout: 15000,
      parallel: false,
      chaos: true,
@@ -258,10 +258,15 @@ Run qualify across all packages in a monorepo workspace:
 apophis qualify --workspace --profile oauth-nightly
 ```

-## `--generation-profile` Flag
+## Test Budget

-Control test data generation depth independently from the qualification profile:
+The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:

-```bash
-apophis qualify --profile oauth-nightly --generation-profile quick
+```javascript
+presets: {
+  'protocol-lab': {
+    runs: 200,
+    timeout: 15000
+  }
+}
 ```
@@ -165,7 +165,7 @@ export default {
  },
  presets: {
    'safe-ci': {
-      depth: 'quick',
+      runs: 10,
      timeout: 5000
    }
  }
@@ -184,10 +184,22 @@ apophis verify --workspace --profile quick --format json

 Output includes per-package pass/fail summaries. Fails if any package fails.

-## `--generation-profile` Flag
+## Test Budget

-Control test data generation depth independently from the verification profile:
+The `runs` field in your preset controls how many property-based tests execute per route. Default is 50. Lower for faster CI feedback, higher for deeper exploration:

-```bash
-apophis verify --profile quick --generation-profile quick
+```javascript
+profiles: {
+  quick: {
+    mode: 'verify',
+    preset: 'safe-ci',
+    routes: ['POST /users']
+  }
+},
+presets: {
+  'safe-ci': {
+    runs: 10,
+    timeout: 5000
+  }
+}
 ```
@@ -50,7 +50,7 @@
    "benchmark:cli": "npm run build && node scripts/bench/cli.mjs",
    "benchmark:hot": "npm run build && node scripts/bench/hot-paths.mjs",
    "profile:qualify": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet",
-    "profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --generation-profile quick --seed 42 --quiet",
+    "profile:qualify:quick": "npm run build && mkdir -p .profiles && node --cpu-prof --cpu-prof-dir=.profiles --cpu-prof-name=qualify-quick.cpuprofile dist/cli/index.js qualify --cwd src/cli/__fixtures__/protocol-lab --profile oauth-nightly --seed 42 --quiet",
    "clean": "rm -rf dist",
    "apophis:verify": "apophis verify --profile quick",
    "apophis:doctor": "apophis doctor"
@@ -8,30 +8,13 @@ const __dirname = fileURLToPath(new URL('.', import.meta.url))
 const repoRoot = resolve(__dirname, '..', '..')

 const options = getBenchOptions()
-const generationProfiles = (process.env.BENCH_GENERATION_PROFILES ?? 'default,quick,standard,thorough')
-  .split(',')
-  .map((value) => value.trim())
-  .filter(Boolean)
-
-function withGenerationProfile(baseArgs, profile) {
-  if (profile === 'default') {
-    return baseArgs
-  }
-  return [...baseArgs, '--generation-profile', profile]
-}

 const scenarios = [
  { name: 'cli.help', args: ['--help'] },
  { name: 'cli.version', args: ['--version'] },
  { name: 'cli.doctor', args: ['doctor', '--cwd', 'src/cli/__fixtures__/tiny-fastify', '--quiet'] },
  { name: 'cli.observe.check', args: ['observe', '--cwd', 'src/cli/__fixtures__/observe-config', '--profile', 'staging-observe', '--check-config', '--quiet'] },
-  ...generationProfiles.map((profile) => ({
-    name: `cli.qualify.profile[${profile}]`,
-    args: withGenerationProfile(
-      ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'],
-      profile,
-    ),
-  })),
+  { name: 'cli.qualify', args: ['qualify', '--cwd', 'src/cli/__fixtures__/protocol-lab', '--profile', 'oauth-nightly', '--seed', '42', '--quiet'] },
 ]

 async function run() {