refactor: remove depth/generationProfile tiering, use explicit runs

- Replace TestDepth ('quick'|'standard'|'thorough') with explicit runs:number
- Remove GenerationProfile type and all generationProfile parameters
- schema-to-arbitrary.ts now uses fixed defaults (string≤128, array≤10, props≤6)
- Delete src/cli/core/generation-profile.ts
- Remove --generation-profile CLI flag from verify and qualify
- Remove depth field from PresetDefinition and all preset scaffolds
- Remove generationProfiles from Config interface
- Update all test files: depth:'quick' → runs:10
- Remove depth from all fixture configs
- Update builders.ts, mutation.ts, outbound-mock-runtime.ts
- Build: clean | Tests: 849 pass, 0 fail
This commit is contained in:
John Dvorak
2026-04-30 13:47:24 -07:00
parent 3e5758dd54
commit 6295c476dc
38 changed files with 130 additions and 526 deletions
@@ -1,7 +1,6 @@
/** /**
* APOPHIS configuration for broken-behavior fixture. * APOPHIS configuration for broken-behavior fixture.
*/ */
export default { export default {
mode: "verify", mode: "verify",
profiles: { profiles: {
@@ -15,7 +14,6 @@ export default {
presets: { presets: {
"safe-ci": { "safe-ci": {
name: "safe-ci", name: "safe-ci",
depth: "quick",
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -2,11 +2,9 @@
* LEGACY APOPHIS configuration (old-style, for migration tests). * LEGACY APOPHIS configuration (old-style, for migration tests).
* This uses deprecated field names that should be detected by `apophis migrate`. * This uses deprecated field names that should be detected by `apophis migrate`.
*/ */
export default { export default {
// Deprecated: 'mode' used to be 'testMode' // Deprecated: 'mode' used to be 'testMode'
testMode: "verify", testMode: "verify",
// Deprecated: 'profiles' used to be 'testProfiles' // Deprecated: 'profiles' used to be 'testProfiles'
testProfiles: { testProfiles: {
quick: { quick: {
@@ -17,7 +15,6 @@ export default {
routeFilter: ["GET /legacy"], routeFilter: ["GET /legacy"],
}, },
}, },
// Deprecated: 'presets' used to be 'testPresets' // Deprecated: 'presets' used to be 'testPresets'
testPresets: { testPresets: {
"safe-ci": { "safe-ci": {
@@ -28,7 +25,6 @@ export default {
maxDuration: 5000, maxDuration: 5000,
}, },
}, },
// Deprecated: 'environments' used to be 'envPolicies' // Deprecated: 'environments' used to be 'envPolicies'
envPolicies: { envPolicies: {
local: { local: {
@@ -2,7 +2,6 @@
* Root-level APOPHIS config for monorepo. * Root-level APOPHIS config for monorepo.
* Packages can override with their own configs. * Packages can override with their own configs.
*/ */
export default { export default {
mode: "verify", mode: "verify",
profiles: { profiles: {
@@ -20,7 +19,6 @@ export default {
presets: { presets: {
"safe-ci": { "safe-ci": {
name: "safe-ci", name: "safe-ci",
depth: "quick",
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -1,7 +1,6 @@
/** /**
* APOPHIS configuration for observe-config fixture. * APOPHIS configuration for observe-config fixture.
*/ */
export default { export default {
mode: "observe", mode: "observe",
profiles: { profiles: {
@@ -15,7 +14,6 @@ export default {
presets: { presets: {
"observe-safe": { "observe-safe": {
name: "observe-safe", name: "observe-safe",
depth: "quick",
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -1,7 +1,6 @@
/** /**
* APOPHIS configuration for protocol-lab fixture. * APOPHIS configuration for protocol-lab fixture.
*/ */
export default { export default {
mode: "qualify", mode: "qualify",
profiles: { profiles: {
@@ -15,7 +14,6 @@ export default {
presets: { presets: {
deep: { deep: {
name: "deep", name: "deep",
depth: "deep",
timeout: 30000, timeout: 30000,
parallel: false, parallel: false,
chaos: true, chaos: true,
@@ -1,7 +1,6 @@
/** /**
* APOPHIS configuration for tiny-fastify fixture. * APOPHIS configuration for tiny-fastify fixture.
*/ */
export default { export default {
mode: "verify", mode: "verify",
profiles: { profiles: {
@@ -15,7 +14,6 @@ export default {
presets: { presets: {
"safe-ci": { "safe-ci": {
name: "safe-ci", name: "safe-ci",
depth: "quick",
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -11,7 +11,6 @@ export default {
presets: { presets: {
"safe-ci": { "safe-ci": {
name: "safe-ci", name: "safe-ci",
depth: "quick",
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -11,7 +11,6 @@ export default {
presets: { presets: {
"safe-ci": { "safe-ci": {
name: "safe-ci", name: "safe-ci",
depth: "quick",
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -11,7 +11,6 @@ export default {
presets: { presets: {
"safe-ci": { "safe-ci": {
name: "safe-ci", name: "safe-ci",
depth: "quick",
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
-4
View File
@@ -17,7 +17,6 @@ export interface ScaffoldResult {
export function safeCiScaffold(): ScaffoldResult { export function safeCiScaffold(): ScaffoldResult {
const preset: PresetDefinition = { const preset: PresetDefinition = {
name: 'safe-ci', name: 'safe-ci',
depth: 'quick',
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -95,7 +94,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
export function platformObserveScaffold(): ScaffoldResult { export function platformObserveScaffold(): ScaffoldResult {
const preset: PresetDefinition = { const preset: PresetDefinition = {
name: 'platform-observe', name: 'platform-observe',
depth: 'standard',
timeout: 10000, timeout: 10000,
parallel: true, parallel: true,
chaos: false, chaos: false,
@@ -180,7 +178,6 @@ This project was scaffolded with \`apophis init --preset platform-observe\`.
export function llmSafeScaffold(): ScaffoldResult { export function llmSafeScaffold(): ScaffoldResult {
const preset: PresetDefinition = { const preset: PresetDefinition = {
name: 'llm-safe', name: 'llm-safe',
depth: 'quick',
timeout: 3000, timeout: 3000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -258,7 +255,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
export function protocolLabScaffold(): ScaffoldResult { export function protocolLabScaffold(): ScaffoldResult {
const preset: PresetDefinition = { const preset: PresetDefinition = {
name: 'protocol-lab', name: 'protocol-lab',
depth: 'deep',
timeout: 15000, timeout: 15000,
parallel: false, parallel: false,
chaos: true, chaos: true,
+1 -26
View File
@@ -19,7 +19,7 @@
import type { CliContext } from '../../core/context.js' import type { CliContext } from '../../core/context.js'
import { loadConfig } from '../../core/config-loader.js' import { loadConfig } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js' import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js' import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js' import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js' import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
@@ -54,13 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
return ROUTE_IDENTITY_PATTERN.test(route) return ROUTE_IDENTITY_PATTERN.test(route)
} }
function coerceDepth(value: unknown): TestConfig['depth'] {
if (value === 'quick' || value === 'standard' || value === 'thorough') {
return value
}
return 'standard'
}
function coerceTimeout(value: unknown): number | undefined { function coerceTimeout(value: unknown): number | undefined {
return typeof value === 'number' ? value : undefined return typeof value === 'number' ? value : undefined
} }
@@ -71,7 +64,6 @@ function coerceTimeout(value: unknown): number | undefined {
export interface QualifyOptions { export interface QualifyOptions {
profile?: string profile?: string
generationProfile?: string
seed?: number seed?: number
config?: string config?: string
cwd?: string cwd?: string
@@ -529,7 +521,6 @@ export async function qualifyCommand(
): Promise<CommandResult> { ): Promise<CommandResult> {
const { const {
profile, profile,
generationProfile,
seed: explicitSeed, seed: explicitSeed,
config: configPath, config: configPath,
cwd, cwd,
@@ -558,7 +549,6 @@ export async function qualifyCommand(
} }
const config = loadResult.config const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2. Run policy engine checks // 2. Run policy engine checks
const policyEngine = new PolicyEngine({ const policyEngine = new PolicyEngine({
@@ -600,12 +590,9 @@ export async function qualifyCommand(
// 6. Build stateful config // 6. Build stateful config
const presetName = profileDef?.preset const presetName = profileDef?.preset
const preset = presetName ? config.presets?.[presetName] : undefined const preset = presetName ? config.presets?.[presetName] : undefined
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout) const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
const statefulConfig: TestConfig | undefined = gates.stateful const statefulConfig: TestConfig | undefined = gates.stateful
? { ? {
depth: presetDepth,
generationProfile: resolvedGenerationProfile,
seed, seed,
timeout: presetTimeout, timeout: presetTimeout,
routes: profileDef?.routes, routes: profileDef?.routes,
@@ -752,12 +739,6 @@ export async function qualifyCommand(
} }
} }
} catch (error) { } catch (error) {
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message: error.message,
}
}
const message = error instanceof Error ? error.message : String(error) const message = error instanceof Error ? error.message : String(error)
return { return {
exitCode: INTERNAL_ERROR, exitCode: INTERNAL_ERROR,
@@ -780,7 +761,6 @@ export async function handleQualify(
): Promise<number> { ): Promise<number> {
const options: QualifyOptions = { const options: QualifyOptions = {
profile: ctx.options.profile || undefined, profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
seed: undefined, seed: undefined,
config: ctx.options.config || undefined, config: ctx.options.config || undefined,
cwd: ctx.cwd, cwd: ctx.cwd,
@@ -798,11 +778,6 @@ export async function handleQualify(
} }
} }
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const result = await qualifyCommand(options, ctx) const result = await qualifyCommand(options, ctx)
const format = options.format || ctx.options.format || 'human' const format = options.format || ctx.options.format || 'human'
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary' const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
+2 -16
View File
@@ -22,7 +22,7 @@
import type { CliContext } from '../../core/context.js' import type { CliContext } from '../../core/context.js'
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js' import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js' import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js' import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js' import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js' import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
@@ -54,7 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
export interface VerifyOptions { export interface VerifyOptions {
profile?: string profile?: string
generationProfile?: string
routes?: string routes?: string
seed?: number seed?: number
changed?: boolean changed?: boolean
@@ -381,7 +380,6 @@ export async function verifyCommand(
): Promise<CommandResult> { ): Promise<CommandResult> {
const { const {
profile, profile,
generationProfile,
routes: routesFlag, routes: routesFlag,
seed: explicitSeed, seed: explicitSeed,
changed, changed,
@@ -412,7 +410,6 @@ export async function verifyCommand(
} }
const config = loadResult.config const config = loadResult.config
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
// 2a. Resolve profile — if explicitly requested but missing, list available ones // 2a. Resolve profile — if explicitly requested but missing, list available ones
if (profile && !config.profiles?.[profile]) { if (profile && !config.profiles?.[profile]) {
@@ -551,12 +548,7 @@ export async function verifyCommand(
message: `Config validation failed: ${message}`, message: `Config validation failed: ${message}`,
} }
} }
if (error instanceof GenerationProfileResolutionError) {
return {
exitCode: USAGE_ERROR,
message,
}
}
return { return {
exitCode: INTERNAL_ERROR, exitCode: INTERNAL_ERROR,
message: `Internal error in verify command: ${message}`, message: `Internal error in verify command: ${message}`,
@@ -578,7 +570,6 @@ export async function handleVerify(
): Promise<number> { ): Promise<number> {
const options: VerifyOptions = { const options: VerifyOptions = {
profile: ctx.options.profile || undefined, profile: ctx.options.profile || undefined,
generationProfile: ctx.options.generationProfile,
routes: undefined, routes: undefined,
seed: undefined, seed: undefined,
changed: false, changed: false,
@@ -610,11 +601,6 @@ export async function handleVerify(
options.changed = true options.changed = true
} }
const generationProfileIdx = args.indexOf('--generation-profile')
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
options.generationProfile = args[generationProfileIdx + 1]
}
const workspaceMode = args.includes('--workspace') const workspaceMode = args.includes('--workspace')
if (workspaceMode) { if (workspaceMode) {
+4 -35
View File
@@ -30,7 +30,6 @@ export interface Config {
environments?: Record<string, EnvironmentPolicy>; environments?: Record<string, EnvironmentPolicy>;
profiles?: Record<string, ProfileDefinition>; profiles?: Record<string, ProfileDefinition>;
presets?: Record<string, PresetDefinition>; presets?: Record<string, PresetDefinition>;
generationProfiles?: Record<string, 'quick' | 'standard' | 'thorough' | { base: 'quick' | 'standard' | 'thorough' }>;
[key: string]: unknown; [key: string]: unknown;
} }
@@ -107,11 +106,6 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
optional: true, optional: true,
properties: {}, properties: {},
}, },
generationProfiles: {
type: 'object',
optional: true,
properties: {},
},
packs: { packs: {
type: 'array', type: 'array',
optional: true, optional: true,
@@ -151,7 +145,6 @@ const PROFILE_SCHEMA: Record<string, SchemaField> = {
// Schema for PresetDefinition values (inside presets.<name>) // Schema for PresetDefinition values (inside presets.<name>)
const PRESET_SCHEMA: Record<string, SchemaField> = { const PRESET_SCHEMA: Record<string, SchemaField> = {
name: { type: 'string', optional: false }, name: { type: 'string', optional: false },
depth: { type: 'string', optional: true, enumValues: ['quick', 'standard', 'deep'] },
timeout: { type: 'number', optional: true, min: 0 }, timeout: { type: 'number', optional: true, min: 0 },
parallel: { type: 'boolean', optional: true }, parallel: { type: 'boolean', optional: true },
chaos: { type: 'boolean', optional: true }, chaos: { type: 'boolean', optional: true },
@@ -160,12 +153,9 @@ const PRESET_SCHEMA: Record<string, SchemaField> = {
sampling: { type: 'number', optional: true }, sampling: { type: 'number', optional: true },
blocking: { type: 'boolean', optional: true }, blocking: { type: 'boolean', optional: true },
sinks: { type: 'object', optional: true }, sinks: { type: 'object', optional: true },
runs: { type: 'number', optional: true, min: 1 },
}; };
const GENERATION_PROFILE_ALIAS_SCHEMA: Record<string, SchemaField> = {
base: { type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Config discovery // Config discovery
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -259,7 +249,6 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
if (path === 'profiles') return PROFILE_SCHEMA; if (path === 'profiles') return PROFILE_SCHEMA;
if (path === 'presets') return PRESET_SCHEMA; if (path === 'presets') return PRESET_SCHEMA;
if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA; if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA;
if (path === 'generationProfiles') return GENERATION_PROFILE_ALIAS_SCHEMA;
return null; return null;
} }
@@ -267,7 +256,7 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
* Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar). * Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar).
*/ */
function isInsideDynamicContainer(path: string): boolean { function isInsideDynamicContainer(path: string): boolean {
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.') || path.startsWith('generationProfiles.'); return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.');
} }
/** /**
@@ -379,18 +368,11 @@ export function validateConfigAgainstSchema(
// Handle dynamic containers: profiles, presets, environments // Handle dynamic containers: profiles, presets, environments
// The keys are user-defined names; their values have specific schemas // The keys are user-defined names; their values have specific schemas
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments' || path === 'generationProfiles'; const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments';
if (!fieldSchema && isDynamicContainer) { if (!fieldSchema && isDynamicContainer) {
const childSchema = getDynamicContainerSchema(path); const childSchema = getDynamicContainerSchema(path);
const fieldValue = obj[key]; const fieldValue = obj[key];
if (path === 'generationProfiles' && typeof fieldValue === 'string') { if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
validateType(
fieldValue,
{ type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
currentPath,
key,
);
} else if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
// Validate the dynamic container value against its specific schema // Validate the dynamic container value against its specific schema
validateConfigAgainstSchema(fieldValue, childSchema, currentPath); validateConfigAgainstSchema(fieldValue, childSchema, currentPath);
} else if (childSchema) { } else if (childSchema) {
@@ -633,19 +615,6 @@ export function validateConfigSemantics(config: Config): void {
); );
} }
} }
if (preset.depth !== undefined) {
const validDepths = ['quick', 'standard', 'deep'];
const depthValue = preset.depth;
if (typeof depthValue === 'string' && !validDepths.includes(depthValue as string)) {
throw new ConfigValidationError(
`Preset "${presetName}" has invalid depth: "${depthValue}"`,
`presets.${presetName}.depth`,
'depth',
depthValue,
`Must be one of: ${validDepths.join(', ')}.`,
);
}
}
} }
} }
-5
View File
@@ -101,10 +101,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
? options.color ? options.color
: 'auto'; : 'auto';
const generationProfile = typeof options.generationProfile === 'string'
? options.generationProfile
: undefined;
return { return {
cwd, cwd,
env: { env: {
@@ -119,7 +115,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
options: { options: {
config: typeof options.config === 'string' ? options.config : undefined, config: typeof options.config === 'string' ? options.config : undefined,
profile: typeof options.profile === 'string' ? options.profile : undefined, profile: typeof options.profile === 'string' ? options.profile : undefined,
generationProfile,
format, format,
color, color,
quiet: options.quiet === true, quiet: options.quiet === true,
-56
View File
@@ -1,56 +0,0 @@
import type { Config } from './config-loader.js'
export type ResolvedGenerationProfile = 'quick' | 'standard' | 'thorough'
export class GenerationProfileResolutionError extends Error {
constructor(message: string) {
super(message)
this.name = 'GenerationProfileResolutionError'
}
}
function isBuiltInProfile(value: string): value is ResolvedGenerationProfile {
return value === 'quick' || value === 'standard' || value === 'thorough' || value === 'deep'
}
function normalizeProfile(value: string): ResolvedGenerationProfile {
if (value === 'deep') return 'thorough'
return value as ResolvedGenerationProfile
}
export function resolveGenerationProfileOverride(
rawProfile: string | undefined,
config: Config,
): ResolvedGenerationProfile | undefined {
if (!rawProfile) {
return undefined
}
if (isBuiltInProfile(rawProfile)) {
return normalizeProfile(rawProfile)
}
const aliases = config.generationProfiles
if (!aliases) {
throw new GenerationProfileResolutionError(
`Unknown generation profile "${rawProfile}". Use one of: quick, standard, deep, or define an alias in config.generationProfiles.`,
)
}
const alias = aliases[rawProfile]
if (!alias) {
const available = Object.keys(aliases).join(', ') || 'none'
throw new GenerationProfileResolutionError(
`Unknown generation profile "${rawProfile}". Built-ins: quick, standard, deep. Config aliases: ${available}.`,
)
}
const target = typeof alias === 'string' ? alias : alias.base
if (!isBuiltInProfile(target)) {
throw new GenerationProfileResolutionError(
`Invalid generation profile alias "${rawProfile}". Alias must resolve to quick, standard, or deep.`,
)
}
return normalizeProfile(target)
}
+2 -9
View File
@@ -22,7 +22,6 @@ const HELP_HEADER = `
${pc.dim('Global Options:')} ${pc.dim('Global Options:')}
--config <path> Config file path --config <path> Config file path
--profile <name> Profile name from config --profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--cwd <path> Working directory override --cwd <path> Working directory override
--format <mode> Output format: human | json | ndjson (default: human) --format <mode> Output format: human | json | ndjson (default: human)
--color <mode> Color mode: auto | always | never (default: auto) --color <mode> Color mode: auto | always | never (default: auto)
@@ -71,7 +70,6 @@ function getCommandHelp(command: string): string {
${pc.dim('Options:')} ${pc.dim('Options:')}
--profile <name> Profile name from config --profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--routes <filter> Route filter pattern --routes <filter> Route filter pattern
--seed <number> Deterministic seed --seed <number> Deterministic seed
--changed Filter to git-modified routes --changed Filter to git-modified routes
@@ -103,7 +101,6 @@ function getCommandHelp(command: string): string {
${pc.dim('Options:')} ${pc.dim('Options:')}
--profile <name> Profile name from config --profile <name> Profile name from config
--generation-profile <name> Generation budget profile (built-in or config alias)
--seed <number> Deterministic seed --seed <number> Deterministic seed
${pc.dim('Examples:')} ${pc.dim('Examples:')}
@@ -225,7 +222,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
// Global flags // Global flags
cli.option('--config <path>', 'Config file path'); cli.option('--config <path>', 'Config file path');
cli.option('--profile <name>', 'Profile name from config'); cli.option('--profile <name>', 'Profile name from config');
cli.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cli.option('--cwd <path>', 'Working directory override'); cli.option('--cwd <path>', 'Working directory override');
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' }); cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' }); cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
@@ -270,7 +266,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
break; break;
case 'verify': case 'verify':
cmd.option('--profile <name>', 'Profile name from config'); cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cmd.option('--routes <filter>', 'Route filter pattern'); cmd.option('--routes <filter>', 'Route filter pattern');
cmd.option('--seed <number>', 'Deterministic seed'); cmd.option('--seed <number>', 'Deterministic seed');
cmd.option('--changed', 'Filter to git-modified routes'); cmd.option('--changed', 'Filter to git-modified routes');
@@ -281,7 +276,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
break; break;
case 'qualify': case 'qualify':
cmd.option('--profile <name>', 'Profile name from config'); cmd.option('--profile <name>', 'Profile name from config');
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
cmd.option('--seed <number>', 'Deterministic seed'); cmd.option('--seed <number>', 'Deterministic seed');
break; break;
case 'replay': case 'replay':
@@ -373,16 +367,15 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
// Handle unknown flags // Handle unknown flags
const knownGlobalFlags = new Set([ const knownGlobalFlags = new Set([
'--config', '--profile', '--cwd', '--format', '--color', '--config', '--profile', '--cwd', '--format', '--color',
'--generation-profile',
'--quiet', '--verbose', '--artifact-dir', '--workspace', '--quiet', '--verbose', '--artifact-dir', '--workspace',
'-v', '--version', '-h', '--help', '-v', '--version', '-h', '--help',
]); ]);
const commandSpecificFlags: Record<string, Set<string>> = { const commandSpecificFlags: Record<string, Set<string>> = {
init: new Set(['--preset', '--force', '--noninteractive']), init: new Set(['--preset', '--force', '--noninteractive']),
verify: new Set(['--profile', '--generation-profile', '--routes', '--seed', '--changed', '--workspace']), verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
observe: new Set(['--profile', '--check-config', '--workspace']), observe: new Set(['--profile', '--check-config', '--workspace']),
qualify: new Set(['--profile', '--generation-profile', '--seed', '--workspace']), qualify: new Set(['--profile', '--seed', '--workspace']),
replay: new Set(['--artifact']), replay: new Set(['--artifact']),
doctor: new Set(['--mode', '--strict', '--workspace']), doctor: new Set(['--mode', '--strict', '--workspace']),
migrate: new Set(['--check', '--dry-run', '--write']), migrate: new Set(['--check', '--dry-run', '--write']),
-3
View File
@@ -29,7 +29,6 @@ export interface CliContext {
options: { options: {
config: string | undefined; config: string | undefined;
profile: string | undefined; profile: string | undefined;
generationProfile?: string;
format: OutputFormat; format: OutputFormat;
color: ColorMode; color: ColorMode;
quiet: boolean; quiet: boolean;
@@ -132,7 +131,6 @@ export interface ProfileDefinition {
* required: ["name"], * required: ["name"],
* properties: { * properties: {
* name: { type: "string" }, * name: { type: "string" },
* depth: { type: "string", enum: ["quick", "standard", "deep"] },
* timeout: { type: "number" }, * timeout: { type: "number" },
* parallel: { type: "boolean" }, * parallel: { type: "boolean" },
* chaos: { type: "boolean" }, * chaos: { type: "boolean" },
@@ -143,7 +141,6 @@ export interface ProfileDefinition {
*/ */
export interface PresetDefinition { export interface PresetDefinition {
name: string; name: string;
depth?: "quick" | "standard" | "deep";
timeout?: number; timeout?: number;
parallel?: boolean; parallel?: boolean;
chaos?: boolean; chaos?: boolean;
+31 -55
View File
@@ -7,13 +7,9 @@ import type { Arbitrary } from 'fast-check'
import * as fc from 'fast-check' import * as fc from 'fast-check'
import { CONTENT_TYPE } from '../infrastructure/http-executor.js' import { CONTENT_TYPE } from '../infrastructure/http-executor.js'
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
export interface SchemaToArbOptions { export interface SchemaToArbOptions {
/** 'request' skips readOnly, 'response' skips writeOnly */ /** 'request' skips readOnly, 'response' skips writeOnly */
readonly context: 'request' | 'response' readonly context: 'request' | 'response'
/** Generation budget profile: quick favors speed, thorough favors breadth */
readonly generationProfile?: GenerationProfile
} }
interface ContextCache { interface ContextCache {
@@ -29,45 +25,32 @@ const patternRegexCache = new Map<string, RegExp>()
const STABLE_SCHEMA_CACHE_LIMIT = 512 const STABLE_SCHEMA_CACHE_LIMIT = 512
const PATTERN_REGEX_CACHE_LIMIT = 256 const PATTERN_REGEX_CACHE_LIMIT = 256
function normalizeProfile(profile: GenerationProfile | undefined): GenerationProfile { // Fixed defaults for generated data size (previously controlled by generationProfile tier)
return profile ?? 'standard' const DEFAULT_STRING_MAX = 128
const DEFAULT_ARRAY_MAX = 10
const DEFAULT_ADDITIONAL_PROPS_MAX = 6
function defaultStringMaxLength(): number | undefined {
return DEFAULT_STRING_MAX
} }
function defaultStringMaxLength(profile: GenerationProfile): number | undefined { function defaultArrayMaxLength(): number | undefined {
if (profile === 'quick') return 48 return DEFAULT_ARRAY_MAX
if (profile === 'standard') return 128
return undefined
} }
function defaultArrayMaxLength(profile: GenerationProfile): number | undefined { function additionalPropsMaxKeys(): number {
if (profile === 'quick') return 4 return DEFAULT_ADDITIONAL_PROPS_MAX
if (profile === 'standard') return 10
return undefined
} }
function additionalPropsMaxKeys(profile: GenerationProfile): number { function buildFallbackAnyArb(): Arbitrary<unknown> {
if (profile === 'quick') return 3
if (profile === 'standard') return 6
return 10
}
function buildFallbackAnyArb(profile: GenerationProfile): Arbitrary<unknown> {
if (profile === 'thorough') {
return fc.anything()
}
const stringMax = profile === 'quick' ? 24 : 64
const arrayMax = profile === 'quick' ? 3 : 6
const dictMax = profile === 'quick' ? 2 : 4
return fc.oneof( return fc.oneof(
fc.constant(null), fc.constant(null),
fc.boolean(), fc.boolean(),
fc.integer(), fc.integer(),
fc.double({ noNaN: true }), fc.double({ noNaN: true }),
fc.string({ maxLength: stringMax }), fc.string({ maxLength: 64 }),
fc.array(fc.string({ maxLength: 16 }), { maxLength: arrayMax }), fc.array(fc.string({ maxLength: 16 }), { maxLength: 6 }),
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: dictMax }), fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: 4 }),
) )
} }
@@ -106,7 +89,6 @@ const getObject = (schema: unknown, key: string): Record<string, unknown> | unde
const buildStringArb = ( const buildStringArb = (
schema: Record<string, unknown>, schema: Record<string, unknown>,
profile: GenerationProfile,
): Arbitrary<string> => { ): Arbitrary<string> => {
const minLength = getNumber(schema, 'minLength') const minLength = getNumber(schema, 'minLength')
const maxLength = getNumber(schema, 'maxLength') const maxLength = getNumber(schema, 'maxLength')
@@ -149,7 +131,7 @@ const buildStringArb = (
if (minLength !== undefined) constraints.minLength = minLength if (minLength !== undefined) constraints.minLength = minLength
if (maxLength !== undefined) constraints.maxLength = maxLength if (maxLength !== undefined) constraints.maxLength = maxLength
else { else {
const capped = defaultStringMaxLength(profile) const capped = defaultStringMaxLength()
if (capped !== undefined) constraints.maxLength = capped if (capped !== undefined) constraints.maxLength = capped
} }
@@ -298,14 +280,13 @@ function getSchemaFingerprint(schema: Record<string, unknown>): string | undefin
function getStableCachedArbitrary( function getStableCachedArbitrary(
schema: Record<string, unknown>, schema: Record<string, unknown>,
context: SchemaToArbOptions['context'], context: SchemaToArbOptions['context'],
profile: GenerationProfile,
): Arbitrary<unknown> | undefined { ): Arbitrary<unknown> | undefined {
const fingerprint = getSchemaFingerprint(schema) const fingerprint = getSchemaFingerprint(schema)
if (!fingerprint) { if (!fingerprint) {
return undefined return undefined
} }
const key = `${context}:${profile}:${fingerprint}` const key = `${context}:${fingerprint}`
const cached = stableSchemaArbitraryCache.get(key) const cached = stableSchemaArbitraryCache.get(key)
if (!cached) { if (!cached) {
return undefined return undefined
@@ -319,7 +300,6 @@ function getStableCachedArbitrary(
function setStableCachedArbitrary( function setStableCachedArbitrary(
schema: Record<string, unknown>, schema: Record<string, unknown>,
context: SchemaToArbOptions['context'], context: SchemaToArbOptions['context'],
profile: GenerationProfile,
arbitrary: Arbitrary<unknown>, arbitrary: Arbitrary<unknown>,
): void { ): void {
const fingerprint = getSchemaFingerprint(schema) const fingerprint = getSchemaFingerprint(schema)
@@ -327,7 +307,7 @@ function setStableCachedArbitrary(
return return
} }
const key = `${context}:${profile}:${fingerprint}` const key = `${context}:${fingerprint}`
if (stableSchemaArbitraryCache.has(key)) { if (stableSchemaArbitraryCache.has(key)) {
stableSchemaArbitraryCache.delete(key) stableSchemaArbitraryCache.delete(key)
} }
@@ -353,19 +333,18 @@ const buildIntegerArb = (schema: Record<string, unknown>): Arbitrary<number> =>
const buildArrayArb = ( const buildArrayArb = (
schema: Record<string, unknown>, schema: Record<string, unknown>,
options: SchemaToArbOptions, options: SchemaToArbOptions,
profile: GenerationProfile,
): Arbitrary<unknown[]> => { ): Arbitrary<unknown[]> => {
const itemsSchema = getObject(schema, 'items') const itemsSchema = getObject(schema, 'items')
const itemArb = itemsSchema !== undefined const itemArb = itemsSchema !== undefined
? convertSchemaInternal(itemsSchema, options, false) ? convertSchemaInternal(itemsSchema, options, false)
: buildFallbackAnyArb(profile) : buildFallbackAnyArb()
const minItems = getNumber(schema, 'minItems') const minItems = getNumber(schema, 'minItems')
const maxItems = getNumber(schema, 'maxItems') const maxItems = getNumber(schema, 'maxItems')
const constraints: { minLength?: number; maxLength?: number } = {} const constraints: { minLength?: number; maxLength?: number } = {}
if (minItems !== undefined) constraints.minLength = minItems if (minItems !== undefined) constraints.minLength = minItems
if (maxItems !== undefined) constraints.maxLength = maxItems if (maxItems !== undefined) constraints.maxLength = maxItems
else { else {
const capped = defaultArrayMaxLength(profile) const capped = defaultArrayMaxLength()
if (capped !== undefined) constraints.maxLength = capped if (capped !== undefined) constraints.maxLength = capped
} }
return fc.array(itemArb, constraints) return fc.array(itemArb, constraints)
@@ -374,7 +353,6 @@ const buildArrayArb = (
const buildObjectArb = ( const buildObjectArb = (
schema: Record<string, unknown>, schema: Record<string, unknown>,
options: SchemaToArbOptions, options: SchemaToArbOptions,
profile: GenerationProfile,
): Arbitrary<Record<string, unknown>> => { ): Arbitrary<Record<string, unknown>> => {
const properties = getObject(schema, 'properties') ?? {} const properties = getObject(schema, 'properties') ?? {}
const required = new Set(getArray(schema, 'required') as string[] ?? []) const required = new Set(getArray(schema, 'required') as string[] ?? [])
@@ -398,14 +376,14 @@ const buildObjectArb = (
const baseArb = fc.record(arbs) const baseArb = fc.record(arbs)
if (additionalProperties === true) { if (additionalProperties === true) {
const extraValueArb = buildFallbackAnyArb(profile) const extraValueArb = buildFallbackAnyArb()
const keyMaxLength = profile === 'quick' ? 16 : 32 const keyMaxLength = 32
return fc.tuple( return fc.tuple(
baseArb, baseArb,
fc.dictionary( fc.dictionary(
fc.string({ maxLength: keyMaxLength }), fc.string({ maxLength: keyMaxLength }),
extraValueArb, extraValueArb,
{ maxKeys: additionalPropsMaxKeys(profile) }, { maxKeys: additionalPropsMaxKeys() },
), ),
).map(([base, extra]) => ({ ).map(([base, extra]) => ({
...base, ...base,
@@ -418,7 +396,6 @@ const buildObjectArb = (
const buildMultipartArb = ( const buildMultipartArb = (
schema: Record<string, unknown>, schema: Record<string, unknown>,
profile: GenerationProfile,
): Arbitrary<{ fields: Record<string, unknown>; files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]> }> => { ): Arbitrary<{ fields: Record<string, unknown>; files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]> }> => {
const fieldsSchema = getObject(schema, 'x-multipart-fields') ?? {} const fieldsSchema = getObject(schema, 'x-multipart-fields') ?? {}
const filesSchema = getObject(schema, 'x-multipart-files') ?? {} const filesSchema = getObject(schema, 'x-multipart-files') ?? {}
@@ -426,7 +403,7 @@ const buildMultipartArb = (
const fieldArbs: Record<string, Arbitrary<unknown>> = {} const fieldArbs: Record<string, Arbitrary<unknown>> = {}
for (const [key, propSchema] of Object.entries(fieldsSchema)) { for (const [key, propSchema] of Object.entries(fieldsSchema)) {
if (isObject(propSchema)) { if (isObject(propSchema)) {
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request', generationProfile: profile }, false) fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request' }, false)
} }
} }
@@ -463,7 +440,6 @@ const convertSchemaInternal = (
options: SchemaToArbOptions, options: SchemaToArbOptions,
useStableCache: boolean, useStableCache: boolean,
): Arbitrary<unknown> => { ): Arbitrary<unknown> => {
const profile = normalizeProfile(options.generationProfile)
const cacheKey = options.context const cacheKey = options.context
const cachedBySchema = schemaArbitraryCache.get(schema) const cachedBySchema = schemaArbitraryCache.get(schema)
const cached = cachedBySchema?.[cacheKey] const cached = cachedBySchema?.[cacheKey]
@@ -472,7 +448,7 @@ const convertSchemaInternal = (
} }
if (useStableCache) { if (useStableCache) {
const stableCached = getStableCachedArbitrary(schema, cacheKey, profile) const stableCached = getStableCachedArbitrary(schema, cacheKey)
if (stableCached) { if (stableCached) {
const contextCache = cachedBySchema ?? {} const contextCache = cachedBySchema ?? {}
contextCache[cacheKey] = stableCached contextCache[cacheKey] = stableCached
@@ -489,11 +465,11 @@ const convertSchemaInternal = (
let arb: Arbitrary<unknown> let arb: Arbitrary<unknown>
if (contentType === CONTENT_TYPE.MULTIPART) { if (contentType === CONTENT_TYPE.MULTIPART) {
arb = buildMultipartArb(schema, profile) arb = buildMultipartArb(schema)
} else if (enumValues !== undefined && enumValues.length > 0) { } else if (enumValues !== undefined && enumValues.length > 0) {
arb = fc.constantFrom(...enumValues) arb = fc.constantFrom(...enumValues)
} else if (type === 'string') { } else if (type === 'string') {
arb = buildStringArb(schema, profile) arb = buildStringArb(schema)
} else if (type === 'integer') { } else if (type === 'integer') {
arb = buildIntegerArb(schema) arb = buildIntegerArb(schema)
} else if (type === 'number') { } else if (type === 'number') {
@@ -501,11 +477,11 @@ const convertSchemaInternal = (
} else if (type === 'boolean') { } else if (type === 'boolean') {
arb = fc.boolean() arb = fc.boolean()
} else if (type === 'array') { } else if (type === 'array') {
arb = buildArrayArb(schema, options, profile) arb = buildArrayArb(schema, options)
} else if (type === 'object') { } else if (type === 'object') {
arb = buildObjectArb(schema, options, profile) arb = buildObjectArb(schema, options)
} else { } else {
arb = buildFallbackAnyArb(profile) arb = buildFallbackAnyArb()
} }
if (nullable === true) { if (nullable === true) {
@@ -516,7 +492,7 @@ const convertSchemaInternal = (
contextCache[cacheKey] = arb contextCache[cacheKey] = arb
schemaArbitraryCache.set(schema, contextCache) schemaArbitraryCache.set(schema, contextCache)
if (useStableCache) { if (useStableCache) {
setStableCachedArbitrary(schema, cacheKey, profile, arb) setStableCachedArbitrary(schema, cacheKey, arb)
} }
return arb return arb
+5 -9
View File
@@ -220,7 +220,6 @@ function setNestedValue(obj: Record<string, unknown>, path: string, value: unkno
function createConditionalDependencyArbitrary( function createConditionalDependencyArbitrary(
contract: ResolvedOutboundContract, contract: ResolvedOutboundContract,
request: Record<string, unknown>, request: Record<string, unknown>,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<DependencyResponseSample> { ): Arbitrary<DependencyResponseSample> {
const statuses = Object.keys(contract.response).map(Number) const statuses = Object.keys(contract.response).map(Number)
if (statuses.length === 0) { if (statuses.length === 0) {
@@ -229,7 +228,7 @@ function createConditionalDependencyArbitrary(
return fc.integer({ min: 0, max: statuses.length - 1 }).chain((statusIndex) => { return fc.integer({ min: 0, max: statuses.length - 1 }).chain((statusIndex) => {
const statusCode = statuses[statusIndex]! const statusCode = statuses[statusIndex]!
const schema = contract.response[statusCode] const schema = contract.response[statusCode]
const bodyArb = convertSchema(schema ?? {}, { context: 'response', generationProfile }) const bodyArb = convertSchema(schema ?? {}, { context: 'response' })
return bodyArb.map((rawBody) => ({ return bodyArb.map((rawBody) => ({
contractName: contract.name, contractName: contract.name,
statusCode, statusCode,
@@ -239,11 +238,10 @@ function createConditionalDependencyArbitrary(
} }
function createRequestArbitrary( function createRequestArbitrary(
route: RouteContract, route: RouteContract,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<Record<string, unknown>> { ): Arbitrary<Record<string, unknown>> {
const bodySchema = route.schema?.body as Record<string, unknown> | undefined const bodySchema = route.schema?.body as Record<string, unknown> | undefined
const bodyArb = bodySchema !== undefined const bodyArb = bodySchema !== undefined
? convertSchema(bodySchema, { context: 'request', generationProfile }) ? convertSchema(bodySchema, { context: 'request' })
: fc.constant({}) : fc.constant({})
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? [] const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {} const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
@@ -264,11 +262,10 @@ function createRequestArbitrary(
function createConditionalDependenciesArbitrary( function createConditionalDependenciesArbitrary(
contracts: ResolvedOutboundContract[], contracts: ResolvedOutboundContract[],
request: Record<string, unknown>, request: Record<string, unknown>,
generationProfile: 'quick' | 'standard' | 'thorough',
): Arbitrary<ReadonlyArray<DependencyResponseSample>> { ): Arbitrary<ReadonlyArray<DependencyResponseSample>> {
if (contracts.length === 0) return fc.constant([]) if (contracts.length === 0) return fc.constant([])
const arbs = contracts.map((contract) => const arbs = contracts.map((contract) =>
createConditionalDependencyArbitrary(contract, request, generationProfile) createConditionalDependencyArbitrary(contract, request)
) )
return fc.tuple(...arbs) return fc.tuple(...arbs)
} }
@@ -292,11 +289,10 @@ export function createTripleBoundaryArbitrary(
route: RouteContract, route: RouteContract,
contracts: ResolvedOutboundContract[], contracts: ResolvedOutboundContract[],
chaosConfig: ChaosConfig, chaosConfig: ChaosConfig,
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
): Arbitrary<TripleBoundaryCommand> { ): Arbitrary<TripleBoundaryCommand> {
const requestArb = createRequestArbitrary(route, generationProfile) const requestArb = createRequestArbitrary(route)
return requestArb.chain((request) => { return requestArb.chain((request) => {
const depArb = createConditionalDependenciesArbitrary(contracts, request, generationProfile) const depArb = createConditionalDependenciesArbitrary(contracts, request)
const chaosArb = createChaosEventArbitrary(route, contracts, chaosConfig) const chaosArb = createChaosEventArbitrary(route, contracts, chaosConfig)
return fc.tuple(depArb, chaosArb).map(([dependencyResponses, chaosEvents]) => ({ return fc.tuple(depArb, chaosArb).map(([dependencyResponses, chaosEvents]) => ({
route, route,
+1 -2
View File
@@ -29,7 +29,6 @@ export interface OutboundMockRuntime {
interface OutboundMockOptions { interface OutboundMockOptions {
readonly contracts: ResolvedOutboundContract[] readonly contracts: ResolvedOutboundContract[]
readonly mode: 'example' | 'property' readonly mode: 'example' | 'property'
readonly generationProfile?: 'quick' | 'standard' | 'thorough'
readonly overrides?: Record<string, { readonly overrides?: Record<string, {
readonly forceStatus?: number readonly forceStatus?: number
readonly headers?: Record<string, string> readonly headers?: Record<string, string>
@@ -77,7 +76,7 @@ export function createOutboundMockRuntime(opts: OutboundMockOptions): OutboundMo
const schema = contract.response[statusCode] const schema = contract.response[statusCode]
if (!schema) return null if (!schema) return null
// Generate base response from schema // Generate base response from schema
const arb = convertSchema(schema, { context: 'response', generationProfile: opts.generationProfile }) const arb = convertSchema(schema, { context: 'response' })
const samples = fc.sample(arb, { numRuns: 1, seed: opts.seed + calls.length }) const samples = fc.sample(arb, { numRuns: 1, seed: opts.seed + calls.length })
let body = samples[0] ?? null let body = samples[0] ?? null
if (typeof body !== 'object' || body === null) return body if (typeof body !== 'object' || body === null) return body
+1 -1
View File
@@ -27,7 +27,7 @@ import { parse } from '../formula/parser.js'
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
const normalizeTestConfig = (opts: TestConfig = {}): TestConfig => ({ const normalizeTestConfig = (opts: TestConfig = {}): TestConfig => ({
depth: opts.depth ?? 'standard', runs: opts.runs,
scope: opts.scope, scope: opts.scope,
seed: opts.seed, seed: opts.seed,
timeout: opts.timeout, timeout: opts.timeout,
-4
View File
@@ -80,7 +80,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
presets: { presets: {
'protocol-lab': { 'protocol-lab': {
name: 'protocol-lab', name: 'protocol-lab',
depth: 'deep',
timeout: opts.timeout ?? 15000, timeout: opts.timeout ?? 15000,
parallel: false, parallel: false,
chaos: true, chaos: true,
@@ -88,7 +87,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
}, },
'safe-ci': { 'safe-ci': {
name: 'safe-ci', name: 'safe-ci',
depth: 'quick',
timeout: opts.timeout ?? 5000, timeout: opts.timeout ?? 5000,
parallel: false, parallel: false,
chaos: false, chaos: false,
@@ -118,7 +116,6 @@ export function rfc8628DeviceAuthorizationPack(opts: PackOptions = {}): Partial<
presets: { presets: {
'protocol-lab': { 'protocol-lab': {
name: 'protocol-lab', name: 'protocol-lab',
depth: 'deep',
timeout: opts.timeout ?? 20000, timeout: opts.timeout ?? 20000,
parallel: false, parallel: false,
chaos: true, chaos: true,
@@ -148,7 +145,6 @@ export function rfc8693TokenExchangePack(opts: PackOptions = {}): Partial<Config
presets: { presets: {
'protocol-lab': { 'protocol-lab': {
name: 'protocol-lab', name: 'protocol-lab',
depth: 'deep',
timeout: opts.timeout ?? 15000, timeout: opts.timeout ?? 15000,
parallel: false, parallel: false,
chaos: true, chaos: true,
+7 -7
View File
@@ -7,7 +7,7 @@
* If the test suite passes, the mutation "survives" — indicating a gap in coverage. * If the test suite passes, the mutation "survives" — indicating a gap in coverage.
* *
* Usage: * Usage:
* const report = await runMutationTesting(fastify, { depth: 'quick' }) * const report = await runMutationTesting(fastify, { runs: 10 })
* console.log(`Mutation score: ${report.score}%`) * console.log(`Mutation score: ${report.score}%`)
*/ */
import type { FastifyInstance } from 'fastify' import type { FastifyInstance } from 'fastify'
@@ -44,7 +44,7 @@ export interface MutationReport {
readonly weakContracts: string[] // contracts that survived all mutations readonly weakContracts: string[] // contracts that survived all mutations
} }
export interface MutationConfig { export interface MutationConfig {
readonly depth?: TestConfig['depth'] readonly runs?: number
readonly seed?: number readonly seed?: number
/** Max mutations per contract (default: 5) */ /** Max mutations per contract (default: 5) */
readonly maxMutationsPerContract?: number readonly maxMutationsPerContract?: number
@@ -214,7 +214,7 @@ export async function runMutationTesting(
const suite = await runPetitTestsWithMutation( const suite = await runPetitTestsWithMutation(
fastify as unknown as FastifyInjectInstance, fastify as unknown as FastifyInjectInstance,
{ {
depth: config.depth ?? 'quick', runs: config.runs ?? 10,
seed: config.seed, seed: config.seed,
}, },
mutatedContract mutatedContract
@@ -260,13 +260,13 @@ export async function runMutationTesting(
*/ */
async function runPetitTestsWithMutation( async function runPetitTestsWithMutation(
fastify: FastifyInjectInstance, fastify: FastifyInjectInstance,
config: { depth?: TestConfig['depth']; seed?: number }, config: { runs?: number; seed?: number },
mutatedContract: RouteContract mutatedContract: RouteContract
): Promise<TestSuite> { ): Promise<TestSuite> {
// For now, run the full suite - the mutated contract will be discovered // For now, run the full suite - the mutated contract will be discovered
// In a real implementation, you'd inject the mutated contract into the discovery // In a real implementation, you'd inject the mutated contract into the discovery
return runPetitTests(fastify, { return runPetitTests(fastify, {
depth: config.depth ?? 'quick', runs: config.runs ?? 10,
seed: config.seed, seed: config.seed,
routes: [`${mutatedContract.method} ${mutatedContract.path}`], routes: [`${mutatedContract.method} ${mutatedContract.path}`],
}) })
@@ -279,14 +279,14 @@ export async function testMutation(
fastify: FastifyInstance, fastify: FastifyInstance,
contract: RouteContract, contract: RouteContract,
mutation: Mutation, mutation: Mutation,
config: Pick<MutationConfig, 'depth' | 'seed'> = {} config: Pick<MutationConfig, 'runs' | 'seed'> = {}
): Promise<boolean> { ): Promise<boolean> {
const mutatedContract = applyMutation(contract, mutation) const mutatedContract = applyMutation(contract, mutation)
try { try {
const suite = await runPetitTestsWithMutation( const suite = await runPetitTestsWithMutation(
fastify as unknown as FastifyInjectInstance, fastify as unknown as FastifyInjectInstance,
{ {
depth: config.depth ?? 'quick', runs: config.runs ?? 10,
seed: config.seed, seed: config.seed,
}, },
mutatedContract mutatedContract
+7 -11
View File
@@ -57,7 +57,7 @@ async function expectLoadConfigError(
test('schema: accepts minimal valid configs', () => { test('schema: accepts minimal valid configs', () => {
validateConfigAgainstSchema({}, CONFIG_SCHEMA); validateConfigAgainstSchema({}, CONFIG_SCHEMA);
validateConfigAgainstSchema({ mode: 'verify', routes: ['GET /users'], seed: 42 }, CONFIG_SCHEMA); validateConfigAgainstSchema({ mode: 'verify', routes: ['GET /users'], seed: 42 }, CONFIG_SCHEMA);
validateConfigAgainstSchema({ presets: { quick: { depth: 'quick', timeout: 5000 } } }, CONFIG_SCHEMA); validateConfigAgainstSchema({ presets: { quick: { runs: 10, timeout: 5000 } } }, CONFIG_SCHEMA);
}); });
test('schema: rejects unknown keys with guidance', () => { test('schema: rejects unknown keys with guidance', () => {
@@ -131,11 +131,7 @@ test('schema: rejects enum and numeric range violations with clear guidance', ()
path: 'profiles.default.mode', path: 'profiles.default.mode',
expectedGuidance: ['verify', 'observe', 'qualify'], expectedGuidance: ['verify', 'observe', 'qualify'],
}, },
{
value: { presets: { quick: { depth: 'super-deep' } } },
path: 'presets.quick.depth',
expectedGuidance: ['quick', 'standard', 'deep'],
},
] as const; ] as const;
for (const c of enumCases) { for (const c of enumCases) {
@@ -215,9 +211,9 @@ test('semantic: validates cross-reference and value rules', () => {
{ value: { seed: 3.14 }, path: 'seed', guidance: 'integer' }, { value: { seed: 3.14 }, path: 'seed', guidance: 'integer' },
{ value: { presets: { quick: { timeout: -100 } } }, path: 'presets.quick.timeout', guidance: 'non-negative' }, { value: { presets: { quick: { timeout: -100 } } }, path: 'presets.quick.timeout', guidance: 'non-negative' },
{ {
value: { presets: { quick: { depth: 'super-deep' } } }, value: { presets: { quick: { timeout: -100 } } },
path: 'presets.quick.depth', path: 'presets.quick.timeout',
guidance: 'quick, standard, deep', guidance: 'non-negative',
}, },
] as const; ] as const;
@@ -232,7 +228,7 @@ test('semantic: validates cross-reference and value rules', () => {
{ routes: ['GET /users', 'POST /items'] }, { routes: ['GET /users', 'POST /items'] },
{ seed: -42 }, { seed: -42 },
{ seed: 0 }, { seed: 0 },
{ presets: { quick: { timeout: 0, depth: 'standard' } } }, { presets: { quick: { timeout: 0 } } },
]; ];
for (const value of acceptCases) { for (const value of acceptCases) {
@@ -322,7 +318,7 @@ test('loadConfig: resolves profile and preset and applies profile overrides', as
}, },
presets: { presets: {
quick: { quick: {
depth: 'quick', runs: 10,
timeout: 5000, timeout: 5000,
parallel: false, parallel: false,
}, },
+1 -164
View File
@@ -23,12 +23,10 @@
* - Property and state model-based testing focused on confidence * - Property and state model-based testing focused on confidence
* - Iterative small steps with rapid feedback loops * - Iterative small steps with rapid feedback loops
*/ */
import { test } from 'node:test'; import { test } from 'node:test';
import assert from 'node:assert'; import assert from 'node:assert';
import { writeFileSync, readFileSync } from 'node:fs'; import { writeFileSync, readFileSync } from 'node:fs';
import { resolve } from 'node:path'; import { resolve } from 'node:path';
import { import {
migrateCommand, migrateCommand,
detectAllLegacyPatterns, detectAllLegacyPatterns,
@@ -36,31 +34,25 @@ import {
type MigrateOptions, type MigrateOptions,
type MigrationItem, type MigrationItem,
} from '../../cli/commands/migrate/index.js'; } from '../../cli/commands/migrate/index.js';
import { import {
rewriteConfigFile, rewriteConfigFile,
detectLegacyConfigFields, detectLegacyConfigFields,
detectLegacyFieldsNoEquivalent, detectLegacyFieldsNoEquivalent,
detectMixedLegacyModernFields, detectMixedLegacyModernFields,
} from '../../cli/commands/migrate/rewriters/config-rewriter.js'; } from '../../cli/commands/migrate/rewriters/config-rewriter.js';
import { import {
rewriteRouteAnnotations, rewriteRouteAnnotations,
detectLegacyRouteAnnotations, detectLegacyRouteAnnotations,
detectAmbiguousRoutePatterns, detectAmbiguousRoutePatterns,
} from '../../cli/commands/migrate/rewriters/route-rewriter.js'; } from '../../cli/commands/migrate/rewriters/route-rewriter.js';
import { import {
rewriteCodePatterns, rewriteCodePatterns,
detectLegacyCodePatterns, detectLegacyCodePatterns,
detectAmbiguousCodePatterns, detectAmbiguousCodePatterns,
} from '../../cli/commands/migrate/rewriters/code-rewriter.js'; } from '../../cli/commands/migrate/rewriters/code-rewriter.js';
import { createTempDir, cleanup, makeCtx } from './helpers.js'; import { createTempDir, cleanup, makeCtx } from './helpers.js';
test('migrate --check detects broad legacy config field set', async () => { test('migrate --check detects broad legacy config field set', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const legacyConfig = `export default { const legacyConfig = `export default {
testMode: "verify", testMode: "verify",
@@ -82,12 +74,9 @@ test('migrate --check detects broad legacy config field set', async () => {
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns are found'); assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns are found');
const legacyNames = result.items.map((item) => item.legacy); const legacyNames = result.items.map((item) => item.legacy);
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode'); assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
@@ -103,29 +92,23 @@ test('migrate --check detects broad legacy config field set', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 1: Mixed legacy and modern config detection // Test 1: Mixed legacy and modern config detection
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate detects mixed legacy and modern config fields', async () => { test('migrate detects mixed legacy and modern config fields', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
// Config with both legacy and modern fields present // Config with both legacy and modern fields present
const mixedConfig = `export default { const mixedConfig = `export default {
// Legacy field // Legacy field
testMode: "verify", testMode: "verify",
// Modern field (conflicts with legacy) // Modern field (conflicts with legacy)
mode: "observe", mode: "observe",
profiles: { profiles: {
quick: { quick: {
preset: "safe-ci", preset: "safe-ci",
}, },
}, },
// Legacy container // Legacy container
testProfiles: { testProfiles: {
old: { old: {
@@ -133,22 +116,17 @@ test('migrate detects mixed legacy and modern config fields', async () => {
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), mixedConfig); writeFileSync(resolve(dir, 'apophis.config.js'), mixedConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
// Should detect legacy patterns // Should detect legacy patterns
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found'); assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
assert.ok(result.items.length > 0, 'Should detect legacy items'); assert.ok(result.items.length > 0, 'Should detect legacy items');
// Check that mixed fields are reported // Check that mixed fields are reported
const legacyNames = result.items.map((item) => item.legacy); const legacyNames = result.items.map((item) => item.legacy);
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode'); assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles'); assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles');
assert.ok(legacyNames.includes('usesPreset'), 'Should detect usesPreset'); assert.ok(legacyNames.includes('usesPreset'), 'Should detect usesPreset');
// Verify guidance mentions the conflict // Verify guidance mentions the conflict
const testModeItem = result.items.find((item) => item.legacy === 'testMode'); const testModeItem = result.items.find((item) => item.legacy === 'testMode');
assert.ok(testModeItem, 'Should have testMode item'); assert.ok(testModeItem, 'Should have testMode item');
@@ -157,19 +135,15 @@ test('migrate detects mixed legacy and modern config fields', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 2: Dry-run shows exact rewrites // Test 2: Dry-run shows exact rewrites
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate dry-run shows exact file path, line number, legacy text, replacement text', async () => { test('migrate dry-run shows exact file path, line number, legacy text, replacement text', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const legacyConfig = `export default { const legacyConfig = `export default {
// Line 2 // Line 2
testMode: "verify", testMode: "verify",
profiles: { profiles: {
quick: { quick: {
// Line 7 // Line 7
@@ -177,36 +151,27 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ dryRun: true }, ctx); const result = await migrateCommand({ dryRun: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found'); assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
assert.ok(result.message, 'Should have output message'); assert.ok(result.message, 'Should have output message');
// Verify dry-run output contains exact details // Verify dry-run output contains exact details
assert.ok(result.message.includes('Dry run'), 'Should indicate dry run'); assert.ok(result.message.includes('Dry run'), 'Should indicate dry run');
assert.ok(result.message.includes('testMode'), 'Should show legacy text'); assert.ok(result.message.includes('testMode'), 'Should show legacy text');
assert.ok(result.message.includes('mode'), 'Should show replacement text'); assert.ok(result.message.includes('mode'), 'Should show replacement text');
assert.ok(result.message.includes('usesPreset'), 'Should show usesPreset'); assert.ok(result.message.includes('usesPreset'), 'Should show usesPreset');
assert.ok(result.message.includes('preset'), 'Should show preset replacement'); assert.ok(result.message.includes('preset'), 'Should show preset replacement');
// Verify file path is shown // Verify file path is shown
assert.ok(result.message.includes('apophis.config.js'), 'Should show file path'); assert.ok(result.message.includes('apophis.config.js'), 'Should show file path');
// Verify line numbers are shown // Verify line numbers are shown
assert.ok(result.message.includes(':2') || result.message.includes(': 2'), 'Should show line number'); assert.ok(result.message.includes(':2') || result.message.includes(': 2'), 'Should show line number');
// Verify total count // Verify total count
assert.ok(result.message.includes('Total:'), 'Should show total count'); assert.ok(result.message.includes('Total:'), 'Should show total count');
assert.ok(result.message.includes('3'), 'Should show correct total (3 items)'); assert.ok(result.message.includes('3'), 'Should show correct total (3 items)');
// Verify files would be modified // Verify files would be modified
assert.ok(result.filesWouldBeModified, 'Should list files that would be modified'); assert.ok(result.filesWouldBeModified, 'Should list files that would be modified');
assert.strictEqual(result.filesWouldBeModified.length, 1, 'Should show 1 file would be modified'); assert.strictEqual(result.filesWouldBeModified.length, 1, 'Should show 1 file would be modified');
// Verify file was NOT modified // Verify file was NOT modified
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8'); const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
assert.ok(content.includes('testMode'), 'File should still have testMode'); assert.ok(content.includes('testMode'), 'File should still have testMode');
@@ -215,14 +180,11 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 3: Write performs rewrites correctly // Test 3: Write performs rewrites correctly
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate write performs rewrites correctly', async () => { test('migrate write performs rewrites correctly', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const legacyConfig = `export default { const legacyConfig = `export default {
testMode: "verify", testMode: "verify",
@@ -232,16 +194,12 @@ test('migrate write performs rewrites correctly', async () => {
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx); const result = await migrateCommand({ write: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when rewrites performed'); assert.strictEqual(result.exitCode, 1, 'Should exit 1 when rewrites performed');
assert.ok(result.completed.length > 0, 'Should have completed items'); assert.ok(result.completed.length > 0, 'Should have completed items');
assert.ok(result.filesModified && result.filesModified.length > 0, 'Should list modified files'); assert.ok(result.filesModified && result.filesModified.length > 0, 'Should list modified files');
// Verify file WAS modified // Verify file WAS modified
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8'); const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
assert.ok(!content.includes('testMode'), 'File should not have testMode'); assert.ok(!content.includes('testMode'), 'File should not have testMode');
@@ -254,35 +212,26 @@ test('migrate write performs rewrites correctly', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 4: Ambiguous rewrite stops and shows context // Test 4: Ambiguous rewrite stops and shows context
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate ambiguous rewrite stops and shows surrounding context', async () => { test('migrate ambiguous rewrite stops and shows surrounding context', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
// Create a file with an ambiguous code pattern // Create a file with an ambiguous code pattern
const ambiguousCode = `import Fastify from 'fastify'; const ambiguousCode = `import Fastify from 'fastify';
const app = Fastify(); const app = Fastify();
// This is ambiguous: what does oldApi() mean here? // This is ambiguous: what does oldApi() mean here?
app.register(oldApi()); app.register(oldApi());
export default app;`; export default app;`;
writeFileSync(resolve(dir, 'app.js'), ambiguousCode); writeFileSync(resolve(dir, 'app.js'), ambiguousCode);
// Also create a config file so migration has something to work with // Also create a config file so migration has something to work with
const config = `export default { const config = `export default {
mode: "verify", mode: "verify",
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), config); writeFileSync(resolve(dir, 'apophis.config.js'), config);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx); const result = await migrateCommand({ write: true }, ctx);
// Should stop with exit code 2 (USAGE_ERROR) because ambiguous patterns found // Should stop with exit code 2 (USAGE_ERROR) because ambiguous patterns found
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode'); assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
assert.ok(result.remaining.length > 0, 'Should have remaining items'); assert.ok(result.remaining.length > 0, 'Should have remaining items');
@@ -290,21 +239,17 @@ export default app;`;
assert.ok(result.message.includes('Ambiguous'), 'Should mention ambiguous patterns'); assert.ok(result.message.includes('Ambiguous'), 'Should mention ambiguous patterns');
assert.ok(result.message.includes('oldApi()'), 'Should show the ambiguous pattern'); assert.ok(result.message.includes('oldApi()'), 'Should show the ambiguous pattern');
assert.ok(result.message.includes('manual choice'), 'Should mention manual choice'); assert.ok(result.message.includes('manual choice'), 'Should mention manual choice');
// Verify context is shown (surrounding lines) // Verify context is shown (surrounding lines)
assert.ok(result.message.includes('app.register'), 'Should show surrounding context'); assert.ok(result.message.includes('app.register'), 'Should show surrounding context');
} finally { } finally {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 5: Legacy field with no equivalent emits guidance // Test 5: Legacy field with no equivalent emits guidance
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate legacy field with no direct equivalent emits human guidance', async () => { test('migrate legacy field with no direct equivalent emits human guidance', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
// Config with a legacy field that has no direct equivalent // Config with a legacy field that has no direct equivalent
const legacyConfig = `export default { const legacyConfig = `export default {
@@ -317,16 +262,12 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
// This field is deprecated with no direct equivalent // This field is deprecated with no direct equivalent
legacyField: true, legacyField: true,
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
// Should detect the legacy field with no equivalent // Should detect the legacy field with no equivalent
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found'); assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
assert.ok(result.items.length > 0, 'Should detect legacy items'); assert.ok(result.items.length > 0, 'Should detect legacy items');
const legacyFieldItem = result.items.find((item) => item.legacy === 'legacyField'); const legacyFieldItem = result.items.find((item) => item.legacy === 'legacyField');
assert.ok(legacyFieldItem, 'Should detect legacyField'); assert.ok(legacyFieldItem, 'Should detect legacyField');
assert.ok(legacyFieldItem.guidance, 'Should have guidance for legacyField'); assert.ok(legacyFieldItem.guidance, 'Should have guidance for legacyField');
@@ -343,14 +284,11 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 6: Partial migration reports completed and remaining // Test 6: Partial migration reports completed and remaining
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate partial migration reports completed and remaining items', async () => { test('migrate partial migration reports completed and remaining items', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const legacyConfig = `export default { const legacyConfig = `export default {
testMode: "verify", testMode: "verify",
@@ -360,12 +298,9 @@ test('migrate partial migration reports completed and remaining items', async ()
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx); const result = await migrateCommand({ write: true }, ctx);
assert.ok(result.completed.length > 0, 'Should have completed items'); assert.ok(result.completed.length > 0, 'Should have completed items');
assert.ok(result.message, 'Should have output message'); assert.ok(result.message, 'Should have output message');
assert.ok(result.message.includes('Completed'), 'Should mention completed'); assert.ok(result.message.includes('Completed'), 'Should mention completed');
@@ -374,20 +309,16 @@ test('migrate partial migration reports completed and remaining items', async ()
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 7: Preserves comments/formatting where feasible // Test 7: Preserves comments/formatting where feasible
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate preserves comments and formatting where feasible', async () => { test('migrate preserves comments and formatting where feasible', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
// Config with specific formatting (comments, indentation) // Config with specific formatting (comments, indentation)
const legacyConfig = `export default { const legacyConfig = `export default {
// This is a comment about testMode // This is a comment about testMode
testMode: "verify", testMode: "verify",
/* /*
* Block comment about testProfiles * Block comment about testProfiles
*/ */
@@ -398,19 +329,14 @@ test('migrate preserves comments and formatting where feasible', async () => {
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx); const result = await migrateCommand({ write: true }, ctx);
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8'); const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
// Verify comments are preserved // Verify comments are preserved
assert.ok(content.includes('// This is a comment about testMode'), 'Should preserve line comment'); assert.ok(content.includes('// This is a comment about testMode'), 'Should preserve line comment');
assert.ok(content.includes('Block comment about testProfiles'), 'Should preserve block comment'); assert.ok(content.includes('Block comment about testProfiles'), 'Should preserve block comment');
assert.ok(content.includes('// Inline comment'), 'Should preserve inline comment'); assert.ok(content.includes('// Inline comment'), 'Should preserve inline comment');
// Verify replacements were made // Verify replacements were made
assert.ok(content.includes('mode:'), 'Should have mode'); assert.ok(content.includes('mode:'), 'Should have mode');
assert.ok(content.includes('profiles:'), 'Should have profiles'); assert.ok(content.includes('profiles:'), 'Should have profiles');
@@ -419,14 +345,11 @@ test('migrate preserves comments and formatting where feasible', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 8: Migrate exits 0 when config is already modern // Test 8: Migrate exits 0 when config is already modern
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate exits 0 when config is already modern', async () => { test('migrate exits 0 when config is already modern', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const modernConfig = `export default { const modernConfig = `export default {
mode: "verify", mode: "verify",
@@ -438,7 +361,7 @@ test('migrate exits 0 when config is already modern', async () => {
}, },
presets: { presets: {
"safe-ci": { "safe-ci": {
depth: "quick", ,
timeout: 5000, timeout: 5000,
}, },
}, },
@@ -448,12 +371,9 @@ test('migrate exits 0 when config is already modern', async () => {
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), modernConfig); writeFileSync(resolve(dir, 'apophis.config.js'), modernConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 0, 'Should exit 0 for modern config'); assert.strictEqual(result.exitCode, 0, 'Should exit 0 for modern config');
assert.strictEqual(result.items.length, 0, 'Should have no items'); assert.strictEqual(result.items.length, 0, 'Should have no items');
assert.ok(result.message, 'Should have message'); assert.ok(result.message, 'Should have message');
@@ -462,35 +382,25 @@ test('migrate exits 0 when config is already modern', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 9: Migrate exits 2 when ambiguous in write mode // Test 9: Migrate exits 2 when ambiguous in write mode
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate exits 2 when ambiguous patterns found in write mode', async () => { test('migrate exits 2 when ambiguous patterns found in write mode', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const config = `export default { const config = `export default {
mode: "verify", mode: "verify",
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), config); writeFileSync(resolve(dir, 'apophis.config.js'), config);
// Create app with an ambiguous pattern // Create app with an ambiguous pattern
const code = `import Fastify from 'fastify'; const code = `import Fastify from 'fastify';
const app = Fastify(); const app = Fastify();
// Ambiguous pattern // Ambiguous pattern
app.register(oldApi()); app.register(oldApi());
export default app;`; export default app;`;
writeFileSync(resolve(dir, 'app.js'), code); writeFileSync(resolve(dir, 'app.js'), code);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ write: true }, ctx); const result = await migrateCommand({ write: true }, ctx);
// Should exit 2 because ambiguous patterns found // Should exit 2 because ambiguous patterns found
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode'); assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
assert.ok(result.remaining.length > 0, 'Should have remaining ambiguous items'); assert.ok(result.remaining.length > 0, 'Should have remaining ambiguous items');
@@ -499,14 +409,11 @@ export default app;`;
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 10: Migrate emits guidance for each legacy field // Test 10: Migrate emits guidance for each legacy field
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate emits guidance for each legacy field', async () => { test('migrate emits guidance for each legacy field', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const legacyConfig = `export default { const legacyConfig = `export default {
testMode: "verify", testMode: "verify",
@@ -516,14 +423,10 @@ test('migrate emits guidance for each legacy field', async () => {
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
assert.ok(result.items.length > 0, 'Should have items'); assert.ok(result.items.length > 0, 'Should have items');
for (const item of result.items) { for (const item of result.items) {
assert.ok(item.guidance, `Item ${item.legacy} should have guidance`); assert.ok(item.guidance, `Item ${item.legacy} should have guidance`);
assert.ok( assert.ok(
@@ -535,14 +438,11 @@ test('migrate emits guidance for each legacy field', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 11: Config rewriter replaces legacy fields // Test 11: Config rewriter replaces legacy fields
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('config rewriter replaces legacy fields', () => { test('config rewriter replaces legacy fields', () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const content = `export default { const content = `export default {
testMode: "verify", testMode: "verify",
@@ -552,17 +452,13 @@ test('config rewriter replaces legacy fields', () => {
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'test.config.js'), content); writeFileSync(resolve(dir, 'test.config.js'), content);
const items = detectLegacyConfigFields(content, 'test.config.js'); const items = detectLegacyConfigFields(content, 'test.config.js');
assert.strictEqual(items.length, 3, 'Should detect 3 legacy fields'); assert.strictEqual(items.length, 3, 'Should detect 3 legacy fields');
const result = rewriteConfigFile( const result = rewriteConfigFile(
resolve(dir, 'test.config.js'), resolve(dir, 'test.config.js'),
items, items,
); );
assert.strictEqual(result.modified, true, 'Should modify content'); assert.strictEqual(result.modified, true, 'Should modify content');
assert.ok(result.content.includes('mode:'), 'Should have mode'); assert.ok(result.content.includes('mode:'), 'Should have mode');
assert.ok(result.content.includes('profiles:'), 'Should have profiles'); assert.ok(result.content.includes('profiles:'), 'Should have profiles');
@@ -572,35 +468,28 @@ test('config rewriter replaces legacy fields', () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 12: Route rewriter detects x-validate-runtime annotation // Test 12: Route rewriter detects x-validate-runtime annotation
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('route rewriter detects x-validate-runtime annotation', () => { test('route rewriter detects x-validate-runtime annotation', () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const content = `export default { const content = `export default {
schema: { schema: {
'x-validate-runtime': true, 'x-validate-runtime': true,
}, },
};`; };`;
writeFileSync(resolve(dir, 'test.routes.js'), content); writeFileSync(resolve(dir, 'test.routes.js'), content);
const items = detectLegacyRouteAnnotations(content, 'test.routes.js'); const items = detectLegacyRouteAnnotations(content, 'test.routes.js');
assert.strictEqual(items.length, 1, 'Should detect 1 legacy annotation'); assert.strictEqual(items.length, 1, 'Should detect 1 legacy annotation');
const firstItem = items[0]; const firstItem = items[0];
assert.ok(firstItem, 'Expected one migration item'); assert.ok(firstItem, 'Expected one migration item');
assert.strictEqual(firstItem.legacy, 'x-validate-runtime'); assert.strictEqual(firstItem.legacy, 'x-validate-runtime');
assert.strictEqual(firstItem.replacement, 'runtime'); assert.strictEqual(firstItem.replacement, 'runtime');
const result = rewriteRouteAnnotations( const result = rewriteRouteAnnotations(
resolve(dir, 'test.routes.js'), resolve(dir, 'test.routes.js'),
items, items,
); );
assert.strictEqual(result.modified, true, 'Should modify content'); assert.strictEqual(result.modified, true, 'Should modify content');
assert.ok(result.content.includes("'runtime'"), 'Should have runtime'); assert.ok(result.content.includes("'runtime'"), 'Should have runtime');
assert.ok(!result.content.includes('x-validate-runtime'), 'Should not have legacy annotation'); assert.ok(!result.content.includes('x-validate-runtime'), 'Should not have legacy annotation');
@@ -608,34 +497,25 @@ test('route rewriter detects x-validate-runtime annotation', () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 13: Code rewriter detects legacy patterns // Test 13: Code rewriter detects legacy patterns
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('code rewriter detects legacy patterns', () => { test('code rewriter detects legacy patterns', () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const content = `import Fastify from 'fastify'; const content = `import Fastify from 'fastify';
const app = Fastify(); const app = Fastify();
app.register(contract()); app.register(contract());
app.register(stateful()); app.register(stateful());
app.register(scenario()); app.register(scenario());
export default app;`; export default app;`;
writeFileSync(resolve(dir, 'test.app.js'), content); writeFileSync(resolve(dir, 'test.app.js'), content);
const items = detectLegacyCodePatterns(content, 'test.app.js'); const items = detectLegacyCodePatterns(content, 'test.app.js');
assert.strictEqual(items.length, 3, 'Should detect 3 legacy patterns'); assert.strictEqual(items.length, 3, 'Should detect 3 legacy patterns');
const result = rewriteCodePatterns( const result = rewriteCodePatterns(
resolve(dir, 'test.app.js'), resolve(dir, 'test.app.js'),
items, items,
); );
assert.strictEqual(result.modified, true, 'Should modify content'); assert.strictEqual(result.modified, true, 'Should modify content');
assert.ok(result.content.includes("verify({ kind: 'contract' })"), 'Should have verify'); assert.ok(result.content.includes("verify({ kind: 'contract' })"), 'Should have verify');
assert.ok(result.content.includes("qualify({ kind: 'stateful' })"), 'Should have qualify stateful'); assert.ok(result.content.includes("qualify({ kind: 'stateful' })"), 'Should have qualify stateful');
@@ -644,28 +524,21 @@ export default app;`;
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 14: Dry-run default mode (safe by default) // Test 14: Dry-run default mode (safe by default)
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate defaults to dry-run mode (safe by default)', async () => { test('migrate defaults to dry-run mode (safe by default)', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const legacyConfig = `export default { const legacyConfig = `export default {
testMode: "verify", testMode: "verify",
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
// No mode specified — should default to dry-run // No mode specified — should default to dry-run
const result = await migrateCommand({}, ctx); const result = await migrateCommand({}, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1 in dry-run mode'); assert.strictEqual(result.exitCode, 1, 'Should exit 1 in dry-run mode');
assert.ok(result.message?.includes('Dry run'), 'Should indicate dry run'); assert.ok(result.message?.includes('Dry run'), 'Should indicate dry run');
// Verify file was NOT modified // Verify file was NOT modified
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8'); const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
assert.ok(content.includes('testMode'), 'File should still have testMode'); assert.ok(content.includes('testMode'), 'File should still have testMode');
@@ -673,38 +546,30 @@ test('migrate defaults to dry-run mode (safe by default)', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 15: Mixed legacy/modern field detection at rewriter level // Test 15: Mixed legacy/modern field detection at rewriter level
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('config rewriter detects mixed legacy and modern fields', () => { test('config rewriter detects mixed legacy and modern fields', () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const content = `export default { const content = `export default {
// Both legacy and modern present // Both legacy and modern present
testMode: "verify", testMode: "verify",
mode: "observe", mode: "observe",
testProfiles: { testProfiles: {
quick: { quick: {
usesPreset: "safe-ci", usesPreset: "safe-ci",
}, },
}, },
profiles: { profiles: {
modern: { modern: {
preset: "safe-ci", preset: "safe-ci",
}, },
}, },
};`; };`;
writeFileSync(resolve(dir, 'test.config.js'), content); writeFileSync(resolve(dir, 'test.config.js'), content);
const mixedReports = detectMixedLegacyModernFields(content, 'test.config.js'); const mixedReports = detectMixedLegacyModernFields(content, 'test.config.js');
assert.ok(mixedReports.length > 0, 'Should detect mixed fields'); assert.ok(mixedReports.length > 0, 'Should detect mixed fields');
const testModeReport = mixedReports.find((r) => r.legacy === 'testMode'); const testModeReport = mixedReports.find((r) => r.legacy === 'testMode');
assert.ok(testModeReport, 'Should report testMode as mixed'); assert.ok(testModeReport, 'Should report testMode as mixed');
assert.ok(testModeReport.guidance.includes('testMode'), 'Guidance should mention testMode'); assert.ok(testModeReport.guidance.includes('testMode'), 'Guidance should mention testMode');
@@ -713,14 +578,11 @@ test('config rewriter detects mixed legacy and modern fields', () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 16: Ambiguous route pattern detection // Test 16: Ambiguous route pattern detection
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('route rewriter detects ambiguous route patterns with context', () => { test('route rewriter detects ambiguous route patterns with context', () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const content = `export default { const content = `export default {
schema: { schema: {
@@ -728,12 +590,9 @@ test('route rewriter detects ambiguous route patterns with context', () => {
'x-validate': true, 'x-validate': true,
}, },
};`; };`;
writeFileSync(resolve(dir, 'test.routes.js'), content); writeFileSync(resolve(dir, 'test.routes.js'), content);
const items = detectAmbiguousRoutePatterns(content, 'test.routes.js'); const items = detectAmbiguousRoutePatterns(content, 'test.routes.js');
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern'); assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
const firstItem = items[0]; const firstItem = items[0];
assert.ok(firstItem, 'Expected one migration item'); assert.ok(firstItem, 'Expected one migration item');
assert.strictEqual(firstItem.legacy, 'x-validate'); assert.strictEqual(firstItem.legacy, 'x-validate');
@@ -744,28 +603,20 @@ test('route rewriter detects ambiguous route patterns with context', () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 17: Ambiguous code pattern detection with context // Test 17: Ambiguous code pattern detection with context
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('code rewriter detects ambiguous code patterns with surrounding context', () => { test('code rewriter detects ambiguous code patterns with surrounding context', () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const content = `import Fastify from 'fastify'; const content = `import Fastify from 'fastify';
const app = Fastify(); const app = Fastify();
// Ambiguous pattern // Ambiguous pattern
app.register(oldApi()); app.register(oldApi());
export default app;`; export default app;`;
writeFileSync(resolve(dir, 'test.app.js'), content); writeFileSync(resolve(dir, 'test.app.js'), content);
const items = detectAmbiguousCodePatterns(content, 'test.app.js'); const items = detectAmbiguousCodePatterns(content, 'test.app.js');
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern'); assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
const firstItem = items[0]; const firstItem = items[0];
assert.ok(firstItem, 'Expected one migration item'); assert.ok(firstItem, 'Expected one migration item');
assert.strictEqual(firstItem.legacy, 'oldApi()'); assert.strictEqual(firstItem.legacy, 'oldApi()');
@@ -777,42 +628,32 @@ export default app;`;
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 18: Legacy fixture detection // Test 18: Legacy fixture detection
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate detects legacy patterns in fixture config', async () => { test('migrate detects legacy patterns in fixture config', async () => {
const ctx = makeCtx({ cwd: 'src/cli/__fixtures__/legacy-config' }); const ctx = makeCtx({ cwd: 'src/cli/__fixtures__/legacy-config' });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should detect legacy patterns in fixture'); assert.strictEqual(result.exitCode, 1, 'Should detect legacy patterns in fixture');
assert.ok(result.items.length > 0, 'Should find legacy items'); assert.ok(result.items.length > 0, 'Should find legacy items');
const legacyNames = result.items.map((item) => item.legacy); const legacyNames = result.items.map((item) => item.legacy);
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode in fixture'); assert.ok(legacyNames.includes('testMode'), 'Should detect testMode in fixture');
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles in fixture'); assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles in fixture');
assert.ok(legacyNames.includes('testPresets'), 'Should detect testPresets in fixture'); assert.ok(legacyNames.includes('testPresets'), 'Should detect testPresets in fixture');
assert.ok(legacyNames.includes('envPolicies'), 'Should detect envPolicies in fixture'); assert.ok(legacyNames.includes('envPolicies'), 'Should detect envPolicies in fixture');
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 19: JSON output format // Test 19: JSON output format
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate outputs JSON format with all fields', async () => { test('migrate outputs JSON format with all fields', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const legacyConfig = `export default { const legacyConfig = `export default {
testMode: "verify", testMode: "verify",
};`; };`;
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig); writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
const ctx = makeCtx({ cwd: dir, options: { ...makeCtx().options, format: 'json' } }); const ctx = makeCtx({ cwd: dir, options: { ...makeCtx().options, format: 'json' } });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 1, 'Should exit 1'); assert.strictEqual(result.exitCode, 1, 'Should exit 1');
assert.ok(result.items.length > 0, 'Should have items'); assert.ok(result.items.length > 0, 'Should have items');
assert.ok(result.totalRewrites, 'Should have totalRewrites'); assert.ok(result.totalRewrites, 'Should have totalRewrites');
@@ -821,18 +662,14 @@ test('migrate outputs JSON format with all fields', async () => {
cleanup(dir); cleanup(dir);
} }
}); });
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Test 20: No files found returns usage error // Test 20: No files found returns usage error
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
test('migrate returns usage error when no files found', async () => { test('migrate returns usage error when no files found', async () => {
const dir = createTempDir(); const dir = createTempDir();
try { try {
const ctx = makeCtx({ cwd: dir }); const ctx = makeCtx({ cwd: dir });
const result = await migrateCommand({ check: true }, ctx); const result = await migrateCommand({ check: true }, ctx);
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when no files found'); assert.strictEqual(result.exitCode, 2, 'Should exit 2 when no files found');
assert.ok(result.message?.includes('No config or app files found'), 'Should mention no files found'); assert.ok(result.message?.includes('No config or app files found'), 'Should mention no files found');
} finally { } finally {
+7 -7
View File
@@ -152,7 +152,7 @@ test('contract runner supports cross-operation APOSTL ensures', async () => {
await fastify.register(apophisPlugin, {}) await fastify.register(apophisPlugin, {})
registerItemApi(fastify) registerItemApi(fastify)
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick', seed: 7 }) const result = await fastify.apophis.contract({ runs: 10, seed: 7 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok) const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0) assert.strictEqual(failures.length, 0)
} finally { } finally {
@@ -169,7 +169,7 @@ test('stateful runner supports cross-operation APOSTL ensures', async () => {
await fastify.register(apophisPlugin, {}) await fastify.register(apophisPlugin, {})
registerItemApi(fastify) registerItemApi(fastify)
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 11 }) const result = await fastify.apophis.stateful({ runs: 10, seed: 11 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok) const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0) assert.strictEqual(failures.length, 0)
} finally { } finally {
@@ -217,7 +217,7 @@ test('contract runner applies chaos injection when configured', async () => {
}, async () => ({ ok: true })) }, async () => ({ ok: true }))
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ const result = await fastify.apophis.contract({
depth: 'quick', runs: 10,
seed: 3, seed: 3,
chaos: { chaos: {
probability: 1, probability: 1,
@@ -241,7 +241,7 @@ test('contract runner supports previous(...) with response_body(this) path place
await fastify.register(apophisPlugin, {}) await fastify.register(apophisPlugin, {})
registerPlanApi(fastify) registerPlanApi(fastify)
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick', seed: 17 }) const result = await fastify.apophis.contract({ runs: 10, seed: 17 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok) const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0) assert.strictEqual(failures.length, 0)
} finally { } finally {
@@ -258,7 +258,7 @@ test('stateful runner supports previous(...) with response_body(this) path place
await fastify.register(apophisPlugin, {}) await fastify.register(apophisPlugin, {})
registerPlanApi(fastify) registerPlanApi(fastify)
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 19 }) const result = await fastify.apophis.stateful({ runs: 10, seed: 19 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok) const failures = result.tests.filter((entry: TestResult) => !entry.ok)
assert.strictEqual(failures.length, 0) assert.strictEqual(failures.length, 0)
} finally { } finally {
@@ -406,7 +406,7 @@ test('contract runner validates hypermedia links with route_exists', async () =>
}) })
registerHypermediaApi(fastify) registerHypermediaApi(fastify)
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick', seed: 23 }) const result = await fastify.apophis.contract({ runs: 10, seed: 23 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok) const failures = result.tests.filter((entry: TestResult) => !entry.ok)
if (failures.length > 0) { if (failures.length > 0) {
console.log('Contract failures:', failures.map((f: TestResult) => ({ console.log('Contract failures:', failures.map((f: TestResult) => ({
@@ -441,7 +441,7 @@ test('stateful runner validates cross-route relationships', async () => {
}) })
registerHypermediaApi(fastify) registerHypermediaApi(fastify)
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 29 }) const result = await fastify.apophis.stateful({ runs: 10, seed: 29 })
const failures = result.tests.filter((entry: TestResult) => !entry.ok) const failures = result.tests.filter((entry: TestResult) => !entry.ok)
if (failures.length > 0) { if (failures.length > 0) {
console.log('Stateful failures:', failures.map((f: TestResult) => ({ console.log('Stateful failures:', failures.map((f: TestResult) => ({
+2 -2
View File
@@ -36,7 +36,7 @@ test('APOPHIS_DEBUG=1 logs requests and responses', async () => {
originalDebug(msg, _obj) originalDebug(msg, _obj)
} }
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have tests') assert.ok(result.tests.length > 0, 'should have tests')
// Should have logged at least one request and one response // Should have logged at least one request and one response
@@ -80,7 +80,7 @@ test('APOPHIS_DEBUG=0 does not log requests', async () => {
originalDebug(msg, _obj) originalDebug(msg, _obj)
} }
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have tests') assert.ok(result.tests.length > 0, 'should have tests')
// Should not have any request/response logs // Should not have any request/response logs
+3 -3
View File
@@ -30,7 +30,7 @@ test('example: minimal API compiles and runs', async () => {
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have test results') assert.ok(result.tests.length > 0, 'should have test results')
console.log('Minimal example:', result.summary) console.log('Minimal example:', result.summary)
} finally { } finally {
@@ -111,7 +111,7 @@ test('example: CRUD API with contracts compiles and runs', async () => {
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have test results') assert.ok(result.tests.length > 0, 'should have test results')
console.log('CRUD example:', result.summary) console.log('CRUD example:', result.summary)
} finally { } finally {
@@ -149,7 +149,7 @@ test('example: prefix registration works', async () => {
const itemContract = contracts.find(c => c.path === '/api/v1/items') const itemContract = contracts.find(c => c.path === '/api/v1/items')
assert.ok(itemContract, 'should discover prefixed route') assert.ok(itemContract, 'should discover prefixed route')
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should run tests on prefixed routes') assert.ok(result.tests.length > 0, 'should run tests on prefixed routes')
} finally { } finally {
await fastify.close() await fastify.close()
+9 -9
View File
@@ -243,7 +243,7 @@ test('petit-runner executes tests against real API', async () => {
] ]
const fastifyWithRoutes = Object.assign(fastify, { routes: mockRoutes }) const fastifyWithRoutes = Object.assign(fastify, { routes: mockRoutes })
const result = await runPetitTests(fastifyWithRoutes as any, { const result = await runPetitTests(fastifyWithRoutes as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: undefined seed: undefined
}) })
@@ -367,7 +367,7 @@ test('full integration: plugin + routes + test execution', async () => {
const createUserContract = contracts.find(c => c.path === '/users' && c.method === 'POST') const createUserContract = contracts.find(c => c.path === '/users' && c.method === 'POST')
assert.ok(createUserContract, 'create user contract should exist') assert.ok(createUserContract, 'create user contract should exist')
assert.strictEqual(createUserContract.category, 'constructor') assert.strictEqual(createUserContract.category, 'constructor')
const testResult = await fastify.apophis.contract({ depth: 'quick' }) const testResult = await fastify.apophis.contract({ runs: 10 })
assert.ok(Array.isArray(testResult.tests), 'tests should be an array') assert.ok(Array.isArray(testResult.tests), 'tests should be an array')
assert.ok(testResult.tests.length > 0, 'tests should not be empty') assert.ok(testResult.tests.length > 0, 'tests should not be empty')
await fastify.apophis.cleanup() await fastify.apophis.cleanup()
@@ -439,7 +439,7 @@ test('mode filtering: stateful mode only runs constructor/mutator routes', async
}, async () => ({ status: 'ok' })) }, async () => ({ status: 'ok' }))
await fastify.ready() await fastify.ready()
// Run in stateful mode // Run in stateful mode
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
// In stateful mode, utility routes should be excluded // In stateful mode, utility routes should be excluded
// The test should only run constructor and mutator routes // The test should only run constructor and mutator routes
assert.ok(Array.isArray(result.tests), 'tests should be an array') assert.ok(Array.isArray(result.tests), 'tests should be an array')
@@ -474,7 +474,7 @@ test('failing contract produces ContractViolation with suggestion', async () =>
return { status: 'created' } // Returns 200, not 201 return { status: 'created' } // Returns 200, not 201
}) })
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
// Find the failing test // Find the failing test
const failingTests = result.tests.filter(t => !t.ok) const failingTests = result.tests.filter(t => !t.ok)
assert.ok(failingTests.length > 0, 'should have at least one failing test') assert.ok(failingTests.length > 0, 'should have at least one failing test')
@@ -647,7 +647,7 @@ test('integration: contract routes option limits tested routes', async () => {
}, async () => ({ ok: true })) }, async () => ({ ok: true }))
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ const result = await fastify.apophis.contract({
depth: 'quick', runs: 10,
routes: ['GET /included'], routes: ['GET /included'],
}) })
const includedTests = result.tests.filter(t => t.name.includes('GET /included')) const includedTests = result.tests.filter(t => t.name.includes('GET /included'))
@@ -673,7 +673,7 @@ test('integration: contract variants are tagged and run in declared order', asyn
}, async () => ({ ok: true })) }, async () => ({ ok: true }))
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ const result = await fastify.apophis.contract({
depth: 'quick', runs: 10,
variants: [ variants: [
{ name: 'json', headers: { accept: 'application/json' } }, { name: 'json', headers: { accept: 'application/json' } },
{ name: 'xml', headers: { accept: 'application/xml' } }, { name: 'xml', headers: { accept: 'application/xml' } },
@@ -715,7 +715,7 @@ test('integration: variant headers override scope headers', async () => {
}, async () => ({ ok: true })) }, async () => ({ ok: true }))
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ const result = await fastify.apophis.contract({
depth: 'quick', runs: 10,
variants: [ variants: [
{ name: 'xml', headers: { accept: 'application/xml' } }, { name: 'xml', headers: { accept: 'application/xml' } },
], ],
@@ -744,7 +744,7 @@ test('integration: route-level x-variants are extracted and executed', async ()
}, async () => ({ ok: true })) }, async () => ({ ok: true }))
await fastify.ready() await fastify.ready()
// No call-site variants; route-level variants should drive execution // No call-site variants; route-level variants should drive execution
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
const jsonTests = result.tests.filter((t) => t.name.includes('[variant:json]')) const jsonTests = result.tests.filter((t) => t.name.includes('[variant:json]'))
const xmlTests = result.tests.filter((t) => t.name.includes('[variant:xml]')) const xmlTests = result.tests.filter((t) => t.name.includes('[variant:xml]'))
assert.ok(jsonTests.length > 0, 'route json variant should produce tests') assert.ok(jsonTests.length > 0, 'route json variant should produce tests')
@@ -781,7 +781,7 @@ test('integration: inferred contracts are guarded by status code', async () => {
return { error: 'not found' } return { error: 'not found' }
}) })
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
// Should pass because the inferred const contract is guarded: // Should pass because the inferred const contract is guarded:
// response_code(this) == 200 => response_body(this).status == "success" // response_code(this) == 200 => response_body(this).status == "success"
// The 404 response doesn't trigger the antecedent, so the implication holds. // The 404 response doesn't trigger the antecedent, so the implication holds.
+7 -10
View File
@@ -7,7 +7,7 @@
import { convertSchema } from '../domain/schema-to-arbitrary.js' import { convertSchema } from '../domain/schema-to-arbitrary.js'
import { lookupCache, storeCache } from '../incremental/cache.js' import { lookupCache, storeCache } from '../incremental/cache.js'
import type { ApiCommand } from '../domain/stateful.js' import type { ApiCommand } from '../domain/stateful.js'
import type { DepthConfig, RouteContract } from '../types.js' import type { RouteContract, RunConfig } from '../types.js'
import * as fc from 'fast-check' import * as fc from 'fast-check'
const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({ const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
@@ -19,11 +19,10 @@ const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
export const generateCommands = ( export const generateCommands = (
routes: RouteContract[], routes: RouteContract[],
depth: DepthConfig, runConfig: RunConfig,
seed?: number, seed?: number,
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
): { commands: ApiCommand[][], cacheHits: number, cacheMisses: number } => { ): { commands: ApiCommand[][], cacheHits: number, cacheMisses: number } => {
const commandsPerRoute = Math.max(1, Math.floor(depth.contractRuns / Math.max(routes.length, 1))) const commandsPerRoute = Math.max(1, Math.floor(runConfig.contractRuns / Math.max(routes.length, 1)))
let cacheHits = 0 let cacheHits = 0
let cacheMisses = 0 let cacheMisses = 0
const allCommands = routes.map((route) => { const allCommands = routes.map((route) => {
@@ -40,7 +39,7 @@ export const generateCommands = (
cacheMisses++ cacheMisses++
const bodySchema = route.schema?.body as Record<string, unknown> | undefined const bodySchema = route.schema?.body as Record<string, unknown> | undefined
const bodyArb = bodySchema !== undefined const bodyArb = bodySchema !== undefined
? convertSchema(bodySchema, { context: 'request', generationProfile }) ? convertSchema(bodySchema, { context: 'request' })
: fc.constant({}) : fc.constant({})
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? [] const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {} const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
@@ -82,7 +81,7 @@ import { buildPetitSuite, filterPetitRoutes } from './route-filter.js'
import { executePetitCommandStep } from './petit-command-step.js' import { executePetitCommandStep } from './petit-command-step.js'
import { runTripleBoundaryPropertyTest } from './triple-boundary-runner.js' import { runTripleBoundaryPropertyTest } from './triple-boundary-runner.js'
import { makeTrackedResource } from '../domain/state-operations.js' import { makeTrackedResource } from '../domain/state-operations.js'
import { resolveDepth, resolveGenerationProfile } from '../types.js' import { resolveRuns } from '../types.js'
import type { import type {
EvalContext, EvalContext,
FastifyInjectInstance, FastifyInjectInstance,
@@ -133,9 +132,8 @@ export const runPetitTests = async (
} }
} }
const depth = resolveDepth(config.depth ?? 'standard') const runConfig = resolveRuns(config.runs)
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth) const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, runConfig, config.seed)
const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, depth, config.seed, generationProfile)
const allCommands = commandGroups.flat() const allCommands = commandGroups.flat()
const rng = config.seed !== undefined ? new SeededRng(config.seed) : undefined const rng = config.seed !== undefined ? new SeededRng(config.seed) : undefined
@@ -181,7 +179,6 @@ export const runPetitTests = async (
? createOutboundMockRuntime({ ? createOutboundMockRuntime({
contracts: outboundContractRegistry.resolve(Array.from(outboundNames)), contracts: outboundContractRegistry.resolve(Array.from(outboundNames)),
mode: config.outboundMocks?.mode ?? 'example', mode: config.outboundMocks?.mode ?? 'example',
generationProfile,
overrides: config.outboundMocks?.overrides, overrides: config.outboundMocks?.overrides,
unmatched: config.outboundMocks?.unmatched ?? 'error', unmatched: config.outboundMocks?.unmatched ?? 'error',
seed: config.seed !== undefined ? hashCombine(config.seed, 0x6d6f636b) : Math.floor(Math.random() * 0xffffffff), seed: config.seed !== undefined ? hashCombine(config.seed, 0x6d6f636b) : Math.floor(Math.random() * 0xffffffff),
+6 -6
View File
@@ -7,7 +7,7 @@ import apophisPlugin from '../index.js'
import type { TestResult } from '../types.js' import type { TestResult } from '../types.js'
type TestFastifyInstance = FastifyInstance & { type TestFastifyInstance = FastifyInstance & {
apophis: { apophis: {
contract: (opts?: { depth?: string; scope?: string; seed?: number }) => Promise<any> contract: (opts?: { runs?: number; scope?: string; seed?: number }) => Promise<any>
spec: () => Record<string, unknown> spec: () => Record<string, unknown>
} }
} }
@@ -44,19 +44,19 @@ test('scope isolation: routes with x-scope are filtered by scope parameter', asy
}, async () => ({ user: true })) }, async () => ({ user: true }))
await fastify.ready() await fastify.ready()
// Test with no scope - should discover all 3 routes // Test with no scope - should discover all 3 routes
const allResult = await fastify.apophis.contract({ depth: 'quick', scope: undefined }) const allResult = await fastify.apophis.contract({ runs: 10, scope: undefined })
const allPaths = new Set(allResult.tests.map((t: TestResult) => t.name.split(' ')[1])) const allPaths = new Set(allResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
assert.ok(allPaths.has('/public'), 'public route should be in all scope') assert.ok(allPaths.has('/public'), 'public route should be in all scope')
assert.ok(allPaths.has('/admin'), 'admin route should be in all scope') assert.ok(allPaths.has('/admin'), 'admin route should be in all scope')
assert.ok(allPaths.has('/user'), 'user route should be in all scope') assert.ok(allPaths.has('/user'), 'user route should be in all scope')
// Test with admin scope - should only get public + admin // Test with admin scope - should only get public + admin
const adminResult = await fastify.apophis.contract({ depth: 'quick', scope: 'admin' }) const adminResult = await fastify.apophis.contract({ runs: 10, scope: 'admin' })
const adminPaths = new Set(adminResult.tests.map((t: TestResult) => t.name.split(' ')[1])) const adminPaths = new Set(adminResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
assert.ok(adminPaths.has('/public'), 'public route should be in admin scope') assert.ok(adminPaths.has('/public'), 'public route should be in admin scope')
assert.ok(adminPaths.has('/admin'), 'admin route should be in admin scope') assert.ok(adminPaths.has('/admin'), 'admin route should be in admin scope')
assert.ok(!adminPaths.has('/user'), 'user route should NOT be in admin scope') assert.ok(!adminPaths.has('/user'), 'user route should NOT be in admin scope')
// Test with user scope - should only get public + user // Test with user scope - should only get public + user
const userResult = await fastify.apophis.contract({ depth: 'quick', scope: 'user' }) const userResult = await fastify.apophis.contract({ runs: 10, scope: 'user' })
const userPaths = new Set(userResult.tests.map((t: TestResult) => t.name.split(' ')[1])) const userPaths = new Set(userResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
assert.ok(userPaths.has('/public'), 'public route should be in user scope') assert.ok(userPaths.has('/public'), 'public route should be in user scope')
assert.ok(!userPaths.has('/admin'), 'admin route should NOT be in user scope') assert.ok(!userPaths.has('/admin'), 'admin route should NOT be in user scope')
@@ -88,7 +88,7 @@ test('scope isolation: scope headers are passed to requests', async () => {
headers: { 'x-custom-header': 'test-value' }, headers: { 'x-custom-header': 'test-value' },
metadata: {} metadata: {}
}) })
await fastify.apophis.contract({ depth: 'quick', scope: 'test' }) await fastify.apophis.contract({ runs: 10, scope: 'test' })
assert.strictEqual(receivedHeaders['x-custom-header'], 'test-value', 'scope header should be passed to request') assert.strictEqual(receivedHeaders['x-custom-header'], 'test-value', 'scope header should be passed to request')
} finally { } finally {
await fastify.close() await fastify.close()
@@ -130,7 +130,7 @@ test('scope isolation: non-matching scope returns empty test suite', async () =>
}, async () => ({ ok: true })) }, async () => ({ ok: true }))
await fastify.ready() await fastify.ready()
// Test with non-matching scope // Test with non-matching scope
const result = await fastify.apophis.contract({ depth: 'quick', scope: 'other' }) const result = await fastify.apophis.contract({ runs: 10, scope: 'other' })
assert.strictEqual(result.tests.length, 0, 'no tests should run for non-matching scope') assert.strictEqual(result.tests.length, 0, 'no tests should run for non-matching scope')
assert.strictEqual(result.summary.passed, 0, 'no tests should pass') assert.strictEqual(result.summary.passed, 0, 'no tests should pass')
assert.strictEqual(result.summary.failed, 0, 'no tests should fail') assert.strictEqual(result.summary.failed, 0, 'no tests should fail')
+2 -2
View File
@@ -32,7 +32,7 @@ test('serverless: fastify.ready() without listen works', async () => {
await fastify.ready() await fastify.ready()
// Should be able to run tests // Should be able to run tests
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should have tests') assert.ok(result.tests.length > 0, 'should have tests')
// Should be able to get spec // Should be able to get spec
@@ -115,7 +115,7 @@ test('serverless: multiple ready() calls are safe', async () => {
// Second ready() should be safe (idempotent) // Second ready() should be safe (idempotent)
await fastify.ready() await fastify.ready()
const result = await fastify.apophis.contract({ depth: 'quick' }) const result = await fastify.apophis.contract({ runs: 10 })
assert.ok(result.tests.length > 0, 'should still work after multiple ready() calls') assert.ok(result.tests.length > 0, 'should still work after multiple ready() calls')
} finally { } finally {
await fastify.close() await fastify.close()
+8 -8
View File
@@ -22,7 +22,7 @@ test('stateful runner handles empty routes', async () => {
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
}) })
@@ -62,7 +62,7 @@ test('stateful runner executes commands', async () => {
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
}) })
@@ -103,7 +103,7 @@ test('stateful runner detects status code violations', async () => {
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
}) })
@@ -139,7 +139,7 @@ test('stateful runner evaluates APOSTL formulas', async () => {
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
}) })
@@ -189,7 +189,7 @@ test('stateful runner tracks resource state', async () => {
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
}) })
@@ -264,7 +264,7 @@ test('stateful runner substitutes path params from resource state', async () =>
await fastify.ready() await fastify.ready()
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
}) })
@@ -308,7 +308,7 @@ test('stateful runner supports config-level variants', async () => {
await fastify.ready() await fastify.ready()
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
variants: [ variants: [
@@ -359,7 +359,7 @@ test('stateful runner supports route-level x-variants', async () => {
await fastify.ready() await fastify.ready()
const result = await runStatefulTests(fastify as any, { const result = await runStatefulTests(fastify as any, {
depth: 'quick', runs: 10,
scope: undefined, scope: undefined,
seed: 42, seed: 42,
}) })
+7 -10
View File
@@ -6,7 +6,7 @@
* generate → execute → validate → update → check-invariants * generate → execute → validate → update → check-invariants
*/ */
import type { ExtensionRegistry } from '../extension/types.js' import type { ExtensionRegistry } from '../extension/types.js'
import { resolveDepth, resolveGenerationProfile } from '../types.js' import { resolveRuns } from '../types.js'
import { discoverRoutes } from '../domain/discovery.js' import { discoverRoutes } from '../domain/discovery.js'
import { convertSchema } from '../domain/schema-to-arbitrary.js' import { convertSchema } from '../domain/schema-to-arbitrary.js'
import { SeededRng } from '../infrastructure/seeded-rng.js' import { SeededRng } from '../infrastructure/seeded-rng.js'
@@ -22,7 +22,7 @@ import type { OutboundContractRegistry } from '../domain/outbound-contracts.js'
import * as fc from 'fast-check' import * as fc from 'fast-check'
import type { ModelState } from '../domain/stateful.js' import type { ModelState } from '../domain/stateful.js'
import type { CleanupManager } from '../infrastructure/cleanup-manager.js' import type { CleanupManager } from '../infrastructure/cleanup-manager.js'
import type { DepthConfig, EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js' import type { EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js'
// Pure: hash helpers for deterministic sub-seeds // Pure: hash helpers for deterministic sub-seeds
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -61,7 +61,6 @@ class ApiOperation implements StatefulApiOperation {
// ============================================================================ // ============================================================================
const createCommandArbitrary = ( const createCommandArbitrary = (
routes: RouteContract[], routes: RouteContract[],
generationProfile: 'quick' | 'standard' | 'thorough',
): { arb: fc.Arbitrary<ApiOperation>, cacheHits: number, cacheMisses: number } => { ): { arb: fc.Arbitrary<ApiOperation>, cacheHits: number, cacheMisses: number } => {
let cacheHits = 0 let cacheHits = 0
let cacheMisses = 0 let cacheMisses = 0
@@ -74,7 +73,7 @@ const createCommandArbitrary = (
cacheMisses++ cacheMisses++
const bodySchema = route.schema?.body as Record<string, unknown> | undefined const bodySchema = route.schema?.body as Record<string, unknown> | undefined
const arb = bodySchema !== undefined const arb = bodySchema !== undefined
? convertSchema(bodySchema, { context: 'request', generationProfile }) ? convertSchema(bodySchema, { context: 'request' })
: fc.constant({}) : fc.constant({})
return arb.map((params) => new ApiOperation(route, params as Record<string, unknown>)) return arb.map((params) => new ApiOperation(route, params as Record<string, unknown>))
}) })
@@ -93,8 +92,7 @@ export const runStatefulTests = async (
outboundContractRegistry?: OutboundContractRegistry outboundContractRegistry?: OutboundContractRegistry
): Promise<TestSuite> => { ): Promise<TestSuite> => {
const startTime = Date.now() const startTime = Date.now()
const depth = resolveDepth(config.depth ?? 'standard') const runConfig = resolveRuns(config.runs)
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
if (extensionRegistry) { if (extensionRegistry) {
await extensionRegistry.runSuiteStartHooks(config) await extensionRegistry.runSuiteStartHooks(config)
} }
@@ -151,7 +149,7 @@ export const runStatefulTests = async (
variantName ? `[variant:${variantName}] ${name}` : name variantName ? `[variant:${variantName}] ${name}` : name
// Get scope headers for test requests // Get scope headers for test requests
const baseScopeHeaders = scopeRegistry?.getHeaders(config.scope ?? null) ?? {} const baseScopeHeaders = scopeRegistry?.getHeaders(config.scope ?? null) ?? {}
const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes, generationProfile) const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes)
let allResults: TestResult[] = [] let allResults: TestResult[] = []
let globalTestId = 0 let globalTestId = 0
// Create seeded RNG for reproducible path param selection // Create seeded RNG for reproducible path param selection
@@ -178,7 +176,6 @@ export const runStatefulTests = async (
suiteMockRuntime = createOutboundMockRuntime({ suiteMockRuntime = createOutboundMockRuntime({
contracts: allResolved, contracts: allResolved,
mode: config.outboundMocks?.mode ?? 'example', mode: config.outboundMocks?.mode ?? 'example',
generationProfile,
overrides: config.outboundMocks?.overrides, overrides: config.outboundMocks?.overrides,
unmatched: config.outboundMocks?.unmatched ?? 'error', unmatched: config.outboundMocks?.unmatched ?? 'error',
seed: outboundSeed, seed: outboundSeed,
@@ -186,7 +183,7 @@ export const runStatefulTests = async (
suiteMockRuntime.install() suiteMockRuntime.install()
} }
// Run property-based stateful tests per variant // Run property-based stateful tests per variant
const numRuns = depth.statefulRuns const numRuns = runConfig.statefulRuns
const seed = config.seed const seed = config.seed
let counterexampleOutput: string | undefined let counterexampleOutput: string | undefined
const hashString = (s: string): number => { const hashString = (s: string): number => {
@@ -276,7 +273,7 @@ export const runStatefulTests = async (
} }
try { try {
const prop = fc.asyncProperty( const prop = fc.asyncProperty(
fc.array(commandArb, { minLength: 1, maxLength: depth.maxCommands }), fc.array(commandArb, { minLength: 1, maxLength: runConfig.maxCommands }),
async (cmds) => { async (cmds) => {
await runSequence(cmds) await runSequence(cmds)
return true return true
+4 -5
View File
@@ -22,7 +22,7 @@ import type {
TestConfig, TestConfig,
TestResult, TestResult,
} from '../types.js' } from '../types.js'
import { resolveDepth, resolveGenerationProfile } from '../types.js' import { resolveRuns } from '../types.js'
export const runTripleBoundaryPropertyTest = async ( export const runTripleBoundaryPropertyTest = async (
route: RouteContract, route: RouteContract,
@@ -39,10 +39,9 @@ export const runTripleBoundaryPropertyTest = async (
if (!config.chaos) return [] if (!config.chaos) return []
const results: TestResult[] = [] const results: TestResult[] = []
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth) const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos)
const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos, generationProfile) const runConfig = resolveRuns(config.runs)
const depth = resolveDepth(config.depth ?? 'standard') const numRuns = Math.max(10, Math.floor(runConfig.contractRuns / 2))
const numRuns = Math.max(10, Math.floor(depth.contractRuns / 2))
const property = fc.asyncProperty(arbitrary, async (cmd) => { const property = fc.asyncProperty(arbitrary, async (cmd) => {
const testId = testIdBase + results.length + 1 const testId = testIdBase + results.length + 1
+2 -5
View File
@@ -39,18 +39,15 @@ export type {
// Formula and test configuration types // Formula and test configuration types
export type { export type {
TestDepth,
TestConfig, TestConfig,
ResolvedOutboundContract, ResolvedOutboundContract,
OutboundChaosConfig, OutboundChaosConfig,
ChaosConfig, ChaosConfig,
DepthConfig, RunConfig,
} from './types/formula.js' } from './types/formula.js'
export { export {
DEPTH_CONFIGS, resolveRuns,
resolveDepth,
resolveGenerationProfile,
} from './types/formula.js' } from './types/formula.js'
// Extension types // Extension types
+1 -1
View File
@@ -201,7 +201,7 @@ export interface ApophisTestDecorations {
// Forward declarations to avoid circular deps — these are defined in sibling modules // Forward declarations to avoid circular deps — these are defined in sibling modules
export interface TestConfig { export interface TestConfig {
readonly depth?: import('./formula.js').TestDepth readonly runs?: number
readonly scope?: string readonly scope?: string
readonly seed?: number readonly seed?: number
readonly timeout?: number readonly timeout?: number
+10 -34
View File
@@ -7,11 +7,8 @@
// Test: Configuration // Test: Configuration
// ============================================================================ // ============================================================================
export type TestDepth = 'quick' | 'standard' | 'thorough' | { runs: number }
export interface TestConfig { export interface TestConfig {
readonly depth?: TestDepth readonly runs?: number
readonly generationProfile?: GenerationProfile
readonly scope?: string readonly scope?: string
readonly seed?: number readonly seed?: number
readonly timeout?: number readonly timeout?: number
@@ -204,49 +201,28 @@ export interface ChaosConfig {
} }
// ============================================================================ // ============================================================================
// Depth Configuration // Test run configuration
// ============================================================================ // ============================================================================
export interface DepthConfig { export interface RunConfig {
readonly contractRuns: number readonly contractRuns: number
readonly propertyRuns: number readonly propertyRuns: number
readonly statefulRuns: number readonly statefulRuns: number
readonly maxCommands: number readonly maxCommands: number
} }
export type GenerationProfile = 'quick' | 'standard' | 'thorough' const DEFAULT_RUNS = 50
export const DEPTH_CONFIGS: Record<'quick' | 'standard' | 'thorough', DepthConfig> = { export function resolveRuns(runs: number | undefined): RunConfig {
quick: { contractRuns: 10, propertyRuns: 50, statefulRuns: 5, maxCommands: 10 }, const r = runs ?? DEFAULT_RUNS
standard: { contractRuns: 50, propertyRuns: 100, statefulRuns: 20, maxCommands: 30 },
thorough: { contractRuns: 200, propertyRuns: 1000, statefulRuns: 100, maxCommands: 50 }
}
export function resolveDepth(depth: TestDepth): DepthConfig {
if (typeof depth === 'string') {
return DEPTH_CONFIGS[depth]
}
return { return {
contractRuns: depth.runs, contractRuns: r,
propertyRuns: depth.runs, propertyRuns: r * 2,
statefulRuns: Math.max(1, Math.floor(depth.runs / 10)), statefulRuns: Math.max(1, Math.floor(r / 10)),
maxCommands: Math.max(5, Math.floor(depth.runs / 5)), maxCommands: Math.max(5, Math.floor(r / 2)),
} }
} }
export function resolveGenerationProfile(depth: TestDepth | undefined): GenerationProfile {
if (depth === undefined) {
return 'standard'
}
if (typeof depth === 'string') {
return depth
}
if (depth.runs <= 25) return 'quick'
if (depth.runs >= 250) return 'thorough'
return 'standard'
}
// ============================================================================ // ============================================================================
// Test: Results // Test: Results
// ============================================================================ // ============================================================================