refactor: remove depth/generationProfile tiering, use explicit runs
- Replace TestDepth ('quick'|'standard'|'thorough') with explicit runs:number
- Remove GenerationProfile type and all generationProfile parameters
- schema-to-arbitrary.ts now uses fixed defaults (string≤128, array≤10, props≤6)
- Delete src/cli/core/generation-profile.ts
- Remove --generation-profile CLI flag from verify and qualify
- Remove depth field from PresetDefinition and all preset scaffolds
- Remove generationProfiles from Config interface
- Update all test files: depth:'quick' → runs:10
- Remove depth from all fixture configs
- Update builders.ts, mutation.ts, outbound-mock-runtime.ts
- Build: clean | Tests: 849 pass, 0 fail
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for broken-behavior fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "verify",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -2,11 +2,9 @@
|
||||
* LEGACY APOPHIS configuration (old-style, for migration tests).
|
||||
* This uses deprecated field names that should be detected by `apophis migrate`.
|
||||
*/
|
||||
|
||||
export default {
|
||||
// Deprecated: 'mode' used to be 'testMode'
|
||||
testMode: "verify",
|
||||
|
||||
// Deprecated: 'profiles' used to be 'testProfiles'
|
||||
testProfiles: {
|
||||
quick: {
|
||||
@@ -17,7 +15,6 @@ export default {
|
||||
routeFilter: ["GET /legacy"],
|
||||
},
|
||||
},
|
||||
|
||||
// Deprecated: 'presets' used to be 'testPresets'
|
||||
testPresets: {
|
||||
"safe-ci": {
|
||||
@@ -28,7 +25,6 @@ export default {
|
||||
maxDuration: 5000,
|
||||
},
|
||||
},
|
||||
|
||||
// Deprecated: 'environments' used to be 'envPolicies'
|
||||
envPolicies: {
|
||||
local: {
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
* Root-level APOPHIS config for monorepo.
|
||||
* Packages can override with their own configs.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "verify",
|
||||
profiles: {
|
||||
@@ -20,7 +19,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for observe-config fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "observe",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
"observe-safe": {
|
||||
name: "observe-safe",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for protocol-lab fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "qualify",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
deep: {
|
||||
name: "deep",
|
||||
depth: "deep",
|
||||
timeout: 30000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
/**
|
||||
* APOPHIS configuration for tiny-fastify fixture.
|
||||
*/
|
||||
|
||||
export default {
|
||||
mode: "verify",
|
||||
profiles: {
|
||||
@@ -15,7 +14,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -11,7 +11,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -11,7 +11,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -11,7 +11,6 @@ export default {
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
name: "safe-ci",
|
||||
depth: "quick",
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
|
||||
@@ -17,7 +17,6 @@ export interface ScaffoldResult {
|
||||
export function safeCiScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'safe-ci',
|
||||
depth: 'quick',
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
@@ -95,7 +94,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
|
||||
export function platformObserveScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'platform-observe',
|
||||
depth: 'standard',
|
||||
timeout: 10000,
|
||||
parallel: true,
|
||||
chaos: false,
|
||||
@@ -180,7 +178,6 @@ This project was scaffolded with \`apophis init --preset platform-observe\`.
|
||||
export function llmSafeScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'llm-safe',
|
||||
depth: 'quick',
|
||||
timeout: 3000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
@@ -258,7 +255,6 @@ If \`apophis verify\` says "No behavioral contracts found", it means your routes
|
||||
export function protocolLabScaffold(): ScaffoldResult {
|
||||
const preset: PresetDefinition = {
|
||||
name: 'protocol-lab',
|
||||
depth: 'deep',
|
||||
timeout: 15000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
import type { CliContext } from '../../core/context.js'
|
||||
import { loadConfig } from '../../core/config-loader.js'
|
||||
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
||||
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
|
||||
|
||||
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
||||
import type { CommandResult, Artifact, FailureRecord } from '../../core/types.js'
|
||||
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
||||
@@ -54,13 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
|
||||
return ROUTE_IDENTITY_PATTERN.test(route)
|
||||
}
|
||||
|
||||
function coerceDepth(value: unknown): TestConfig['depth'] {
|
||||
if (value === 'quick' || value === 'standard' || value === 'thorough') {
|
||||
return value
|
||||
}
|
||||
return 'standard'
|
||||
}
|
||||
|
||||
function coerceTimeout(value: unknown): number | undefined {
|
||||
return typeof value === 'number' ? value : undefined
|
||||
}
|
||||
@@ -71,7 +64,6 @@ function coerceTimeout(value: unknown): number | undefined {
|
||||
|
||||
export interface QualifyOptions {
|
||||
profile?: string
|
||||
generationProfile?: string
|
||||
seed?: number
|
||||
config?: string
|
||||
cwd?: string
|
||||
@@ -529,7 +521,6 @@ export async function qualifyCommand(
|
||||
): Promise<CommandResult> {
|
||||
const {
|
||||
profile,
|
||||
generationProfile,
|
||||
seed: explicitSeed,
|
||||
config: configPath,
|
||||
cwd,
|
||||
@@ -558,7 +549,6 @@ export async function qualifyCommand(
|
||||
}
|
||||
|
||||
const config = loadResult.config
|
||||
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
|
||||
|
||||
// 2. Run policy engine checks
|
||||
const policyEngine = new PolicyEngine({
|
||||
@@ -600,12 +590,9 @@ export async function qualifyCommand(
|
||||
// 6. Build stateful config
|
||||
const presetName = profileDef?.preset
|
||||
const preset = presetName ? config.presets?.[presetName] : undefined
|
||||
const presetDepth = coerceDepth((preset as { depth?: unknown } | undefined)?.depth)
|
||||
const presetTimeout = coerceTimeout((preset as { timeout?: unknown } | undefined)?.timeout)
|
||||
const statefulConfig: TestConfig | undefined = gates.stateful
|
||||
? {
|
||||
depth: presetDepth,
|
||||
generationProfile: resolvedGenerationProfile,
|
||||
seed,
|
||||
timeout: presetTimeout,
|
||||
routes: profileDef?.routes,
|
||||
@@ -752,12 +739,6 @@ export async function qualifyCommand(
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof GenerationProfileResolutionError) {
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message: error.message,
|
||||
}
|
||||
}
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
return {
|
||||
exitCode: INTERNAL_ERROR,
|
||||
@@ -780,7 +761,6 @@ export async function handleQualify(
|
||||
): Promise<number> {
|
||||
const options: QualifyOptions = {
|
||||
profile: ctx.options.profile || undefined,
|
||||
generationProfile: ctx.options.generationProfile,
|
||||
seed: undefined,
|
||||
config: ctx.options.config || undefined,
|
||||
cwd: ctx.cwd,
|
||||
@@ -798,11 +778,6 @@ export async function handleQualify(
|
||||
}
|
||||
}
|
||||
|
||||
const generationProfileIdx = args.indexOf('--generation-profile')
|
||||
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
|
||||
options.generationProfile = args[generationProfileIdx + 1]
|
||||
}
|
||||
|
||||
const result = await qualifyCommand(options, ctx)
|
||||
const format = options.format || ctx.options.format || 'human'
|
||||
const machineMode = format === 'json' || format === 'ndjson' || format === 'json-summary' || format === 'ndjson-summary'
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
import type { CliContext } from '../../core/context.js'
|
||||
import { loadConfig, findWorkspacePackages } from '../../core/config-loader.js'
|
||||
import { PolicyEngine, detectEnvironment } from '../../core/policy-engine.js'
|
||||
import { resolveGenerationProfileOverride, GenerationProfileResolutionError } from '../../core/generation-profile.js'
|
||||
|
||||
import { SUCCESS, BEHAVIORAL_FAILURE, USAGE_ERROR, INTERNAL_ERROR } from '../../core/exit-codes.js'
|
||||
import type { CommandResult, Artifact, FailureRecord, RouteResult, WorkspaceRun, WorkspaceResult } from '../../core/types.js'
|
||||
import { classifyError, ErrorTaxonomy } from '../../core/error-taxonomy.js'
|
||||
@@ -54,7 +54,6 @@ function isReplayCompatibleRoute(route: string): boolean {
|
||||
|
||||
export interface VerifyOptions {
|
||||
profile?: string
|
||||
generationProfile?: string
|
||||
routes?: string
|
||||
seed?: number
|
||||
changed?: boolean
|
||||
@@ -381,7 +380,6 @@ export async function verifyCommand(
|
||||
): Promise<CommandResult> {
|
||||
const {
|
||||
profile,
|
||||
generationProfile,
|
||||
routes: routesFlag,
|
||||
seed: explicitSeed,
|
||||
changed,
|
||||
@@ -412,7 +410,6 @@ export async function verifyCommand(
|
||||
}
|
||||
|
||||
const config = loadResult.config
|
||||
const resolvedGenerationProfile = resolveGenerationProfileOverride(generationProfile, config)
|
||||
|
||||
// 2a. Resolve profile — if explicitly requested but missing, list available ones
|
||||
if (profile && !config.profiles?.[profile]) {
|
||||
@@ -551,12 +548,7 @@ export async function verifyCommand(
|
||||
message: `Config validation failed: ${message}`,
|
||||
}
|
||||
}
|
||||
if (error instanceof GenerationProfileResolutionError) {
|
||||
return {
|
||||
exitCode: USAGE_ERROR,
|
||||
message,
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
exitCode: INTERNAL_ERROR,
|
||||
message: `Internal error in verify command: ${message}`,
|
||||
@@ -578,7 +570,6 @@ export async function handleVerify(
|
||||
): Promise<number> {
|
||||
const options: VerifyOptions = {
|
||||
profile: ctx.options.profile || undefined,
|
||||
generationProfile: ctx.options.generationProfile,
|
||||
routes: undefined,
|
||||
seed: undefined,
|
||||
changed: false,
|
||||
@@ -610,11 +601,6 @@ export async function handleVerify(
|
||||
options.changed = true
|
||||
}
|
||||
|
||||
const generationProfileIdx = args.indexOf('--generation-profile')
|
||||
if (generationProfileIdx !== -1 && args[generationProfileIdx + 1]) {
|
||||
options.generationProfile = args[generationProfileIdx + 1]
|
||||
}
|
||||
|
||||
const workspaceMode = args.includes('--workspace')
|
||||
|
||||
if (workspaceMode) {
|
||||
|
||||
@@ -30,7 +30,6 @@ export interface Config {
|
||||
environments?: Record<string, EnvironmentPolicy>;
|
||||
profiles?: Record<string, ProfileDefinition>;
|
||||
presets?: Record<string, PresetDefinition>;
|
||||
generationProfiles?: Record<string, 'quick' | 'standard' | 'thorough' | { base: 'quick' | 'standard' | 'thorough' }>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
@@ -107,11 +106,6 @@ const CONFIG_SCHEMA: Record<string, SchemaField> = {
|
||||
optional: true,
|
||||
properties: {},
|
||||
},
|
||||
generationProfiles: {
|
||||
type: 'object',
|
||||
optional: true,
|
||||
properties: {},
|
||||
},
|
||||
packs: {
|
||||
type: 'array',
|
||||
optional: true,
|
||||
@@ -151,7 +145,6 @@ const PROFILE_SCHEMA: Record<string, SchemaField> = {
|
||||
// Schema for PresetDefinition values (inside presets.<name>)
|
||||
const PRESET_SCHEMA: Record<string, SchemaField> = {
|
||||
name: { type: 'string', optional: false },
|
||||
depth: { type: 'string', optional: true, enumValues: ['quick', 'standard', 'deep'] },
|
||||
timeout: { type: 'number', optional: true, min: 0 },
|
||||
parallel: { type: 'boolean', optional: true },
|
||||
chaos: { type: 'boolean', optional: true },
|
||||
@@ -160,12 +153,9 @@ const PRESET_SCHEMA: Record<string, SchemaField> = {
|
||||
sampling: { type: 'number', optional: true },
|
||||
blocking: { type: 'boolean', optional: true },
|
||||
sinks: { type: 'object', optional: true },
|
||||
runs: { type: 'number', optional: true, min: 1 },
|
||||
};
|
||||
|
||||
const GENERATION_PROFILE_ALIAS_SCHEMA: Record<string, SchemaField> = {
|
||||
base: { type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config discovery
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -259,7 +249,6 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
|
||||
if (path === 'profiles') return PROFILE_SCHEMA;
|
||||
if (path === 'presets') return PRESET_SCHEMA;
|
||||
if (path === 'environments') return ENVIRONMENT_POLICY_SCHEMA;
|
||||
if (path === 'generationProfiles') return GENERATION_PROFILE_ALIAS_SCHEMA;
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -267,7 +256,7 @@ function getDynamicContainerSchema(path: string): Record<string, SchemaField> |
|
||||
* Check if a path is inside a dynamic container (e.g., profiles.foo, presets.bar).
|
||||
*/
|
||||
function isInsideDynamicContainer(path: string): boolean {
|
||||
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.') || path.startsWith('generationProfiles.');
|
||||
return path.startsWith('profiles.') || path.startsWith('presets.') || path.startsWith('environments.');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -379,18 +368,11 @@ export function validateConfigAgainstSchema(
|
||||
|
||||
// Handle dynamic containers: profiles, presets, environments
|
||||
// The keys are user-defined names; their values have specific schemas
|
||||
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments' || path === 'generationProfiles';
|
||||
const isDynamicContainer = path === 'profiles' || path === 'presets' || path === 'environments';
|
||||
if (!fieldSchema && isDynamicContainer) {
|
||||
const childSchema = getDynamicContainerSchema(path);
|
||||
const fieldValue = obj[key];
|
||||
if (path === 'generationProfiles' && typeof fieldValue === 'string') {
|
||||
validateType(
|
||||
fieldValue,
|
||||
{ type: 'string', optional: false, enumValues: ['quick', 'standard', 'thorough'] },
|
||||
currentPath,
|
||||
key,
|
||||
);
|
||||
} else if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
|
||||
if (childSchema && fieldValue !== null && typeof fieldValue === 'object') {
|
||||
// Validate the dynamic container value against its specific schema
|
||||
validateConfigAgainstSchema(fieldValue, childSchema, currentPath);
|
||||
} else if (childSchema) {
|
||||
@@ -633,19 +615,6 @@ export function validateConfigSemantics(config: Config): void {
|
||||
);
|
||||
}
|
||||
}
|
||||
if (preset.depth !== undefined) {
|
||||
const validDepths = ['quick', 'standard', 'deep'];
|
||||
const depthValue = preset.depth;
|
||||
if (typeof depthValue === 'string' && !validDepths.includes(depthValue as string)) {
|
||||
throw new ConfigValidationError(
|
||||
`Preset "${presetName}" has invalid depth: "${depthValue}"`,
|
||||
`presets.${presetName}.depth`,
|
||||
'depth',
|
||||
depthValue,
|
||||
`Must be one of: ${validDepths.join(', ')}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -101,10 +101,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
|
||||
? options.color
|
||||
: 'auto';
|
||||
|
||||
const generationProfile = typeof options.generationProfile === 'string'
|
||||
? options.generationProfile
|
||||
: undefined;
|
||||
|
||||
return {
|
||||
cwd,
|
||||
env: {
|
||||
@@ -119,7 +115,6 @@ export function createContext(options: Record<string, unknown> = {}): CliContext
|
||||
options: {
|
||||
config: typeof options.config === 'string' ? options.config : undefined,
|
||||
profile: typeof options.profile === 'string' ? options.profile : undefined,
|
||||
generationProfile,
|
||||
format,
|
||||
color,
|
||||
quiet: options.quiet === true,
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
import type { Config } from './config-loader.js'
|
||||
|
||||
export type ResolvedGenerationProfile = 'quick' | 'standard' | 'thorough'
|
||||
|
||||
export class GenerationProfileResolutionError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message)
|
||||
this.name = 'GenerationProfileResolutionError'
|
||||
}
|
||||
}
|
||||
|
||||
function isBuiltInProfile(value: string): value is ResolvedGenerationProfile {
|
||||
return value === 'quick' || value === 'standard' || value === 'thorough' || value === 'deep'
|
||||
}
|
||||
|
||||
function normalizeProfile(value: string): ResolvedGenerationProfile {
|
||||
if (value === 'deep') return 'thorough'
|
||||
return value as ResolvedGenerationProfile
|
||||
}
|
||||
|
||||
export function resolveGenerationProfileOverride(
|
||||
rawProfile: string | undefined,
|
||||
config: Config,
|
||||
): ResolvedGenerationProfile | undefined {
|
||||
if (!rawProfile) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
if (isBuiltInProfile(rawProfile)) {
|
||||
return normalizeProfile(rawProfile)
|
||||
}
|
||||
|
||||
const aliases = config.generationProfiles
|
||||
if (!aliases) {
|
||||
throw new GenerationProfileResolutionError(
|
||||
`Unknown generation profile "${rawProfile}". Use one of: quick, standard, deep, or define an alias in config.generationProfiles.`,
|
||||
)
|
||||
}
|
||||
|
||||
const alias = aliases[rawProfile]
|
||||
if (!alias) {
|
||||
const available = Object.keys(aliases).join(', ') || 'none'
|
||||
throw new GenerationProfileResolutionError(
|
||||
`Unknown generation profile "${rawProfile}". Built-ins: quick, standard, deep. Config aliases: ${available}.`,
|
||||
)
|
||||
}
|
||||
|
||||
const target = typeof alias === 'string' ? alias : alias.base
|
||||
if (!isBuiltInProfile(target)) {
|
||||
throw new GenerationProfileResolutionError(
|
||||
`Invalid generation profile alias "${rawProfile}". Alias must resolve to quick, standard, or deep.`,
|
||||
)
|
||||
}
|
||||
|
||||
return normalizeProfile(target)
|
||||
}
|
||||
@@ -22,7 +22,6 @@ const HELP_HEADER = `
|
||||
${pc.dim('Global Options:')}
|
||||
--config <path> Config file path
|
||||
--profile <name> Profile name from config
|
||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
||||
--cwd <path> Working directory override
|
||||
--format <mode> Output format: human | json | ndjson (default: human)
|
||||
--color <mode> Color mode: auto | always | never (default: auto)
|
||||
@@ -71,7 +70,6 @@ function getCommandHelp(command: string): string {
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--profile <name> Profile name from config
|
||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
||||
--routes <filter> Route filter pattern
|
||||
--seed <number> Deterministic seed
|
||||
--changed Filter to git-modified routes
|
||||
@@ -103,7 +101,6 @@ function getCommandHelp(command: string): string {
|
||||
|
||||
${pc.dim('Options:')}
|
||||
--profile <name> Profile name from config
|
||||
--generation-profile <name> Generation budget profile (built-in or config alias)
|
||||
--seed <number> Deterministic seed
|
||||
|
||||
${pc.dim('Examples:')}
|
||||
@@ -225,7 +222,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
// Global flags
|
||||
cli.option('--config <path>', 'Config file path');
|
||||
cli.option('--profile <name>', 'Profile name from config');
|
||||
cli.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
||||
cli.option('--cwd <path>', 'Working directory override');
|
||||
cli.option('--format <mode>', 'Output format: human | json | ndjson', { default: 'human' });
|
||||
cli.option('--color <mode>', 'Color mode: auto | always | never', { default: 'auto' });
|
||||
@@ -270,7 +266,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
break;
|
||||
case 'verify':
|
||||
cmd.option('--profile <name>', 'Profile name from config');
|
||||
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
||||
cmd.option('--routes <filter>', 'Route filter pattern');
|
||||
cmd.option('--seed <number>', 'Deterministic seed');
|
||||
cmd.option('--changed', 'Filter to git-modified routes');
|
||||
@@ -281,7 +276,6 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
break;
|
||||
case 'qualify':
|
||||
cmd.option('--profile <name>', 'Profile name from config');
|
||||
cmd.option('--generation-profile <name>', 'Generation budget profile (built-in or config alias)');
|
||||
cmd.option('--seed <number>', 'Deterministic seed');
|
||||
break;
|
||||
case 'replay':
|
||||
@@ -373,16 +367,15 @@ export async function main(argv: string[] = process.argv.slice(2)): Promise<numb
|
||||
// Handle unknown flags
|
||||
const knownGlobalFlags = new Set([
|
||||
'--config', '--profile', '--cwd', '--format', '--color',
|
||||
'--generation-profile',
|
||||
'--quiet', '--verbose', '--artifact-dir', '--workspace',
|
||||
'-v', '--version', '-h', '--help',
|
||||
]);
|
||||
|
||||
const commandSpecificFlags: Record<string, Set<string>> = {
|
||||
init: new Set(['--preset', '--force', '--noninteractive']),
|
||||
verify: new Set(['--profile', '--generation-profile', '--routes', '--seed', '--changed', '--workspace']),
|
||||
verify: new Set(['--profile', '--routes', '--seed', '--changed', '--workspace']),
|
||||
observe: new Set(['--profile', '--check-config', '--workspace']),
|
||||
qualify: new Set(['--profile', '--generation-profile', '--seed', '--workspace']),
|
||||
qualify: new Set(['--profile', '--seed', '--workspace']),
|
||||
replay: new Set(['--artifact']),
|
||||
doctor: new Set(['--mode', '--strict', '--workspace']),
|
||||
migrate: new Set(['--check', '--dry-run', '--write']),
|
||||
|
||||
@@ -29,7 +29,6 @@ export interface CliContext {
|
||||
options: {
|
||||
config: string | undefined;
|
||||
profile: string | undefined;
|
||||
generationProfile?: string;
|
||||
format: OutputFormat;
|
||||
color: ColorMode;
|
||||
quiet: boolean;
|
||||
@@ -132,7 +131,6 @@ export interface ProfileDefinition {
|
||||
* required: ["name"],
|
||||
* properties: {
|
||||
* name: { type: "string" },
|
||||
* depth: { type: "string", enum: ["quick", "standard", "deep"] },
|
||||
* timeout: { type: "number" },
|
||||
* parallel: { type: "boolean" },
|
||||
* chaos: { type: "boolean" },
|
||||
@@ -143,7 +141,6 @@ export interface ProfileDefinition {
|
||||
*/
|
||||
export interface PresetDefinition {
|
||||
name: string;
|
||||
depth?: "quick" | "standard" | "deep";
|
||||
timeout?: number;
|
||||
parallel?: boolean;
|
||||
chaos?: boolean;
|
||||
|
||||
@@ -7,13 +7,9 @@ import type { Arbitrary } from 'fast-check'
|
||||
import * as fc from 'fast-check'
|
||||
import { CONTENT_TYPE } from '../infrastructure/http-executor.js'
|
||||
|
||||
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
|
||||
|
||||
export interface SchemaToArbOptions {
|
||||
/** 'request' skips readOnly, 'response' skips writeOnly */
|
||||
readonly context: 'request' | 'response'
|
||||
/** Generation budget profile: quick favors speed, thorough favors breadth */
|
||||
readonly generationProfile?: GenerationProfile
|
||||
}
|
||||
|
||||
interface ContextCache {
|
||||
@@ -29,45 +25,32 @@ const patternRegexCache = new Map<string, RegExp>()
|
||||
const STABLE_SCHEMA_CACHE_LIMIT = 512
|
||||
const PATTERN_REGEX_CACHE_LIMIT = 256
|
||||
|
||||
function normalizeProfile(profile: GenerationProfile | undefined): GenerationProfile {
|
||||
return profile ?? 'standard'
|
||||
// Fixed defaults for generated data size (previously controlled by generationProfile tier)
|
||||
const DEFAULT_STRING_MAX = 128
|
||||
const DEFAULT_ARRAY_MAX = 10
|
||||
const DEFAULT_ADDITIONAL_PROPS_MAX = 6
|
||||
|
||||
function defaultStringMaxLength(): number | undefined {
|
||||
return DEFAULT_STRING_MAX
|
||||
}
|
||||
|
||||
function defaultStringMaxLength(profile: GenerationProfile): number | undefined {
|
||||
if (profile === 'quick') return 48
|
||||
if (profile === 'standard') return 128
|
||||
return undefined
|
||||
function defaultArrayMaxLength(): number | undefined {
|
||||
return DEFAULT_ARRAY_MAX
|
||||
}
|
||||
|
||||
function defaultArrayMaxLength(profile: GenerationProfile): number | undefined {
|
||||
if (profile === 'quick') return 4
|
||||
if (profile === 'standard') return 10
|
||||
return undefined
|
||||
function additionalPropsMaxKeys(): number {
|
||||
return DEFAULT_ADDITIONAL_PROPS_MAX
|
||||
}
|
||||
|
||||
function additionalPropsMaxKeys(profile: GenerationProfile): number {
|
||||
if (profile === 'quick') return 3
|
||||
if (profile === 'standard') return 6
|
||||
return 10
|
||||
}
|
||||
|
||||
function buildFallbackAnyArb(profile: GenerationProfile): Arbitrary<unknown> {
|
||||
if (profile === 'thorough') {
|
||||
return fc.anything()
|
||||
}
|
||||
|
||||
const stringMax = profile === 'quick' ? 24 : 64
|
||||
const arrayMax = profile === 'quick' ? 3 : 6
|
||||
const dictMax = profile === 'quick' ? 2 : 4
|
||||
|
||||
function buildFallbackAnyArb(): Arbitrary<unknown> {
|
||||
return fc.oneof(
|
||||
fc.constant(null),
|
||||
fc.boolean(),
|
||||
fc.integer(),
|
||||
fc.double({ noNaN: true }),
|
||||
fc.string({ maxLength: stringMax }),
|
||||
fc.array(fc.string({ maxLength: 16 }), { maxLength: arrayMax }),
|
||||
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: dictMax }),
|
||||
fc.string({ maxLength: 64 }),
|
||||
fc.array(fc.string({ maxLength: 16 }), { maxLength: 6 }),
|
||||
fc.dictionary(fc.string({ maxLength: 16 }), fc.string({ maxLength: 24 }), { maxKeys: 4 }),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -106,7 +89,6 @@ const getObject = (schema: unknown, key: string): Record<string, unknown> | unde
|
||||
|
||||
const buildStringArb = (
|
||||
schema: Record<string, unknown>,
|
||||
profile: GenerationProfile,
|
||||
): Arbitrary<string> => {
|
||||
const minLength = getNumber(schema, 'minLength')
|
||||
const maxLength = getNumber(schema, 'maxLength')
|
||||
@@ -149,7 +131,7 @@ const buildStringArb = (
|
||||
if (minLength !== undefined) constraints.minLength = minLength
|
||||
if (maxLength !== undefined) constraints.maxLength = maxLength
|
||||
else {
|
||||
const capped = defaultStringMaxLength(profile)
|
||||
const capped = defaultStringMaxLength()
|
||||
if (capped !== undefined) constraints.maxLength = capped
|
||||
}
|
||||
|
||||
@@ -298,14 +280,13 @@ function getSchemaFingerprint(schema: Record<string, unknown>): string | undefin
|
||||
function getStableCachedArbitrary(
|
||||
schema: Record<string, unknown>,
|
||||
context: SchemaToArbOptions['context'],
|
||||
profile: GenerationProfile,
|
||||
): Arbitrary<unknown> | undefined {
|
||||
const fingerprint = getSchemaFingerprint(schema)
|
||||
if (!fingerprint) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const key = `${context}:${profile}:${fingerprint}`
|
||||
const key = `${context}:${fingerprint}`
|
||||
const cached = stableSchemaArbitraryCache.get(key)
|
||||
if (!cached) {
|
||||
return undefined
|
||||
@@ -319,7 +300,6 @@ function getStableCachedArbitrary(
|
||||
function setStableCachedArbitrary(
|
||||
schema: Record<string, unknown>,
|
||||
context: SchemaToArbOptions['context'],
|
||||
profile: GenerationProfile,
|
||||
arbitrary: Arbitrary<unknown>,
|
||||
): void {
|
||||
const fingerprint = getSchemaFingerprint(schema)
|
||||
@@ -327,7 +307,7 @@ function setStableCachedArbitrary(
|
||||
return
|
||||
}
|
||||
|
||||
const key = `${context}:${profile}:${fingerprint}`
|
||||
const key = `${context}:${fingerprint}`
|
||||
if (stableSchemaArbitraryCache.has(key)) {
|
||||
stableSchemaArbitraryCache.delete(key)
|
||||
}
|
||||
@@ -353,19 +333,18 @@ const buildIntegerArb = (schema: Record<string, unknown>): Arbitrary<number> =>
|
||||
const buildArrayArb = (
|
||||
schema: Record<string, unknown>,
|
||||
options: SchemaToArbOptions,
|
||||
profile: GenerationProfile,
|
||||
): Arbitrary<unknown[]> => {
|
||||
const itemsSchema = getObject(schema, 'items')
|
||||
const itemArb = itemsSchema !== undefined
|
||||
? convertSchemaInternal(itemsSchema, options, false)
|
||||
: buildFallbackAnyArb(profile)
|
||||
: buildFallbackAnyArb()
|
||||
const minItems = getNumber(schema, 'minItems')
|
||||
const maxItems = getNumber(schema, 'maxItems')
|
||||
const constraints: { minLength?: number; maxLength?: number } = {}
|
||||
if (minItems !== undefined) constraints.minLength = minItems
|
||||
if (maxItems !== undefined) constraints.maxLength = maxItems
|
||||
else {
|
||||
const capped = defaultArrayMaxLength(profile)
|
||||
const capped = defaultArrayMaxLength()
|
||||
if (capped !== undefined) constraints.maxLength = capped
|
||||
}
|
||||
return fc.array(itemArb, constraints)
|
||||
@@ -374,7 +353,6 @@ const buildArrayArb = (
|
||||
const buildObjectArb = (
|
||||
schema: Record<string, unknown>,
|
||||
options: SchemaToArbOptions,
|
||||
profile: GenerationProfile,
|
||||
): Arbitrary<Record<string, unknown>> => {
|
||||
const properties = getObject(schema, 'properties') ?? {}
|
||||
const required = new Set(getArray(schema, 'required') as string[] ?? [])
|
||||
@@ -398,14 +376,14 @@ const buildObjectArb = (
|
||||
const baseArb = fc.record(arbs)
|
||||
|
||||
if (additionalProperties === true) {
|
||||
const extraValueArb = buildFallbackAnyArb(profile)
|
||||
const keyMaxLength = profile === 'quick' ? 16 : 32
|
||||
const extraValueArb = buildFallbackAnyArb()
|
||||
const keyMaxLength = 32
|
||||
return fc.tuple(
|
||||
baseArb,
|
||||
fc.dictionary(
|
||||
fc.string({ maxLength: keyMaxLength }),
|
||||
extraValueArb,
|
||||
{ maxKeys: additionalPropsMaxKeys(profile) },
|
||||
{ maxKeys: additionalPropsMaxKeys() },
|
||||
),
|
||||
).map(([base, extra]) => ({
|
||||
...base,
|
||||
@@ -418,7 +396,6 @@ const buildObjectArb = (
|
||||
|
||||
const buildMultipartArb = (
|
||||
schema: Record<string, unknown>,
|
||||
profile: GenerationProfile,
|
||||
): Arbitrary<{ fields: Record<string, unknown>; files: Record<string, { originalname: string; mimetype: string; size: number; buffer: Buffer } | { originalname: string; mimetype: string; size: number; buffer: Buffer }[]> }> => {
|
||||
const fieldsSchema = getObject(schema, 'x-multipart-fields') ?? {}
|
||||
const filesSchema = getObject(schema, 'x-multipart-files') ?? {}
|
||||
@@ -426,7 +403,7 @@ const buildMultipartArb = (
|
||||
const fieldArbs: Record<string, Arbitrary<unknown>> = {}
|
||||
for (const [key, propSchema] of Object.entries(fieldsSchema)) {
|
||||
if (isObject(propSchema)) {
|
||||
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request', generationProfile: profile }, false)
|
||||
fieldArbs[key] = convertSchemaInternal(propSchema, { context: 'request' }, false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -463,7 +440,6 @@ const convertSchemaInternal = (
|
||||
options: SchemaToArbOptions,
|
||||
useStableCache: boolean,
|
||||
): Arbitrary<unknown> => {
|
||||
const profile = normalizeProfile(options.generationProfile)
|
||||
const cacheKey = options.context
|
||||
const cachedBySchema = schemaArbitraryCache.get(schema)
|
||||
const cached = cachedBySchema?.[cacheKey]
|
||||
@@ -472,7 +448,7 @@ const convertSchemaInternal = (
|
||||
}
|
||||
|
||||
if (useStableCache) {
|
||||
const stableCached = getStableCachedArbitrary(schema, cacheKey, profile)
|
||||
const stableCached = getStableCachedArbitrary(schema, cacheKey)
|
||||
if (stableCached) {
|
||||
const contextCache = cachedBySchema ?? {}
|
||||
contextCache[cacheKey] = stableCached
|
||||
@@ -489,11 +465,11 @@ const convertSchemaInternal = (
|
||||
let arb: Arbitrary<unknown>
|
||||
|
||||
if (contentType === CONTENT_TYPE.MULTIPART) {
|
||||
arb = buildMultipartArb(schema, profile)
|
||||
arb = buildMultipartArb(schema)
|
||||
} else if (enumValues !== undefined && enumValues.length > 0) {
|
||||
arb = fc.constantFrom(...enumValues)
|
||||
} else if (type === 'string') {
|
||||
arb = buildStringArb(schema, profile)
|
||||
arb = buildStringArb(schema)
|
||||
} else if (type === 'integer') {
|
||||
arb = buildIntegerArb(schema)
|
||||
} else if (type === 'number') {
|
||||
@@ -501,11 +477,11 @@ const convertSchemaInternal = (
|
||||
} else if (type === 'boolean') {
|
||||
arb = fc.boolean()
|
||||
} else if (type === 'array') {
|
||||
arb = buildArrayArb(schema, options, profile)
|
||||
arb = buildArrayArb(schema, options)
|
||||
} else if (type === 'object') {
|
||||
arb = buildObjectArb(schema, options, profile)
|
||||
arb = buildObjectArb(schema, options)
|
||||
} else {
|
||||
arb = buildFallbackAnyArb(profile)
|
||||
arb = buildFallbackAnyArb()
|
||||
}
|
||||
|
||||
if (nullable === true) {
|
||||
@@ -516,7 +492,7 @@ const convertSchemaInternal = (
|
||||
contextCache[cacheKey] = arb
|
||||
schemaArbitraryCache.set(schema, contextCache)
|
||||
if (useStableCache) {
|
||||
setStableCachedArbitrary(schema, cacheKey, profile, arb)
|
||||
setStableCachedArbitrary(schema, cacheKey, arb)
|
||||
}
|
||||
|
||||
return arb
|
||||
|
||||
@@ -220,7 +220,6 @@ function setNestedValue(obj: Record<string, unknown>, path: string, value: unkno
|
||||
function createConditionalDependencyArbitrary(
|
||||
contract: ResolvedOutboundContract,
|
||||
request: Record<string, unknown>,
|
||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
||||
): Arbitrary<DependencyResponseSample> {
|
||||
const statuses = Object.keys(contract.response).map(Number)
|
||||
if (statuses.length === 0) {
|
||||
@@ -229,7 +228,7 @@ function createConditionalDependencyArbitrary(
|
||||
return fc.integer({ min: 0, max: statuses.length - 1 }).chain((statusIndex) => {
|
||||
const statusCode = statuses[statusIndex]!
|
||||
const schema = contract.response[statusCode]
|
||||
const bodyArb = convertSchema(schema ?? {}, { context: 'response', generationProfile })
|
||||
const bodyArb = convertSchema(schema ?? {}, { context: 'response' })
|
||||
return bodyArb.map((rawBody) => ({
|
||||
contractName: contract.name,
|
||||
statusCode,
|
||||
@@ -239,11 +238,10 @@ function createConditionalDependencyArbitrary(
|
||||
}
|
||||
function createRequestArbitrary(
|
||||
route: RouteContract,
|
||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
||||
): Arbitrary<Record<string, unknown>> {
|
||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||
const bodyArb = bodySchema !== undefined
|
||||
? convertSchema(bodySchema, { context: 'request', generationProfile })
|
||||
? convertSchema(bodySchema, { context: 'request' })
|
||||
: fc.constant({})
|
||||
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
|
||||
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
|
||||
@@ -264,11 +262,10 @@ function createRequestArbitrary(
|
||||
function createConditionalDependenciesArbitrary(
|
||||
contracts: ResolvedOutboundContract[],
|
||||
request: Record<string, unknown>,
|
||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
||||
): Arbitrary<ReadonlyArray<DependencyResponseSample>> {
|
||||
if (contracts.length === 0) return fc.constant([])
|
||||
const arbs = contracts.map((contract) =>
|
||||
createConditionalDependencyArbitrary(contract, request, generationProfile)
|
||||
createConditionalDependencyArbitrary(contract, request)
|
||||
)
|
||||
return fc.tuple(...arbs)
|
||||
}
|
||||
@@ -292,11 +289,10 @@ export function createTripleBoundaryArbitrary(
|
||||
route: RouteContract,
|
||||
contracts: ResolvedOutboundContract[],
|
||||
chaosConfig: ChaosConfig,
|
||||
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
|
||||
): Arbitrary<TripleBoundaryCommand> {
|
||||
const requestArb = createRequestArbitrary(route, generationProfile)
|
||||
const requestArb = createRequestArbitrary(route)
|
||||
return requestArb.chain((request) => {
|
||||
const depArb = createConditionalDependenciesArbitrary(contracts, request, generationProfile)
|
||||
const depArb = createConditionalDependenciesArbitrary(contracts, request)
|
||||
const chaosArb = createChaosEventArbitrary(route, contracts, chaosConfig)
|
||||
return fc.tuple(depArb, chaosArb).map(([dependencyResponses, chaosEvents]) => ({
|
||||
route,
|
||||
|
||||
@@ -29,7 +29,6 @@ export interface OutboundMockRuntime {
|
||||
interface OutboundMockOptions {
|
||||
readonly contracts: ResolvedOutboundContract[]
|
||||
readonly mode: 'example' | 'property'
|
||||
readonly generationProfile?: 'quick' | 'standard' | 'thorough'
|
||||
readonly overrides?: Record<string, {
|
||||
readonly forceStatus?: number
|
||||
readonly headers?: Record<string, string>
|
||||
@@ -77,7 +76,7 @@ export function createOutboundMockRuntime(opts: OutboundMockOptions): OutboundMo
|
||||
const schema = contract.response[statusCode]
|
||||
if (!schema) return null
|
||||
// Generate base response from schema
|
||||
const arb = convertSchema(schema, { context: 'response', generationProfile: opts.generationProfile })
|
||||
const arb = convertSchema(schema, { context: 'response' })
|
||||
const samples = fc.sample(arb, { numRuns: 1, seed: opts.seed + calls.length })
|
||||
let body = samples[0] ?? null
|
||||
if (typeof body !== 'object' || body === null) return body
|
||||
|
||||
@@ -27,7 +27,7 @@ import { parse } from '../formula/parser.js'
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const normalizeTestConfig = (opts: TestConfig = {}): TestConfig => ({
|
||||
depth: opts.depth ?? 'standard',
|
||||
runs: opts.runs,
|
||||
scope: opts.scope,
|
||||
seed: opts.seed,
|
||||
timeout: opts.timeout,
|
||||
|
||||
@@ -80,7 +80,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
|
||||
presets: {
|
||||
'protocol-lab': {
|
||||
name: 'protocol-lab',
|
||||
depth: 'deep',
|
||||
timeout: opts.timeout ?? 15000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
@@ -88,7 +87,6 @@ export function oauth21ProfilePack(opts: PackOptions = {}): Partial<Config> {
|
||||
},
|
||||
'safe-ci': {
|
||||
name: 'safe-ci',
|
||||
depth: 'quick',
|
||||
timeout: opts.timeout ?? 5000,
|
||||
parallel: false,
|
||||
chaos: false,
|
||||
@@ -118,7 +116,6 @@ export function rfc8628DeviceAuthorizationPack(opts: PackOptions = {}): Partial<
|
||||
presets: {
|
||||
'protocol-lab': {
|
||||
name: 'protocol-lab',
|
||||
depth: 'deep',
|
||||
timeout: opts.timeout ?? 20000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
@@ -148,7 +145,6 @@ export function rfc8693TokenExchangePack(opts: PackOptions = {}): Partial<Config
|
||||
presets: {
|
||||
'protocol-lab': {
|
||||
name: 'protocol-lab',
|
||||
depth: 'deep',
|
||||
timeout: opts.timeout ?? 15000,
|
||||
parallel: false,
|
||||
chaos: true,
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* If the test suite passes, the mutation "survives" — indicating a gap in coverage.
|
||||
*
|
||||
* Usage:
|
||||
* const report = await runMutationTesting(fastify, { depth: 'quick' })
|
||||
* const report = await runMutationTesting(fastify, { runs: 10 })
|
||||
* console.log(`Mutation score: ${report.score}%`)
|
||||
*/
|
||||
import type { FastifyInstance } from 'fastify'
|
||||
@@ -44,7 +44,7 @@ export interface MutationReport {
|
||||
readonly weakContracts: string[] // contracts that survived all mutations
|
||||
}
|
||||
export interface MutationConfig {
|
||||
readonly depth?: TestConfig['depth']
|
||||
readonly runs?: number
|
||||
readonly seed?: number
|
||||
/** Max mutations per contract (default: 5) */
|
||||
readonly maxMutationsPerContract?: number
|
||||
@@ -214,7 +214,7 @@ export async function runMutationTesting(
|
||||
const suite = await runPetitTestsWithMutation(
|
||||
fastify as unknown as FastifyInjectInstance,
|
||||
{
|
||||
depth: config.depth ?? 'quick',
|
||||
runs: config.runs ?? 10,
|
||||
seed: config.seed,
|
||||
},
|
||||
mutatedContract
|
||||
@@ -260,13 +260,13 @@ export async function runMutationTesting(
|
||||
*/
|
||||
async function runPetitTestsWithMutation(
|
||||
fastify: FastifyInjectInstance,
|
||||
config: { depth?: TestConfig['depth']; seed?: number },
|
||||
config: { runs?: number; seed?: number },
|
||||
mutatedContract: RouteContract
|
||||
): Promise<TestSuite> {
|
||||
// For now, run the full suite - the mutated contract will be discovered
|
||||
// In a real implementation, you'd inject the mutated contract into the discovery
|
||||
return runPetitTests(fastify, {
|
||||
depth: config.depth ?? 'quick',
|
||||
runs: config.runs ?? 10,
|
||||
seed: config.seed,
|
||||
routes: [`${mutatedContract.method} ${mutatedContract.path}`],
|
||||
})
|
||||
@@ -279,14 +279,14 @@ export async function testMutation(
|
||||
fastify: FastifyInstance,
|
||||
contract: RouteContract,
|
||||
mutation: Mutation,
|
||||
config: Pick<MutationConfig, 'depth' | 'seed'> = {}
|
||||
config: Pick<MutationConfig, 'runs' | 'seed'> = {}
|
||||
): Promise<boolean> {
|
||||
const mutatedContract = applyMutation(contract, mutation)
|
||||
try {
|
||||
const suite = await runPetitTestsWithMutation(
|
||||
fastify as unknown as FastifyInjectInstance,
|
||||
{
|
||||
depth: config.depth ?? 'quick',
|
||||
runs: config.runs ?? 10,
|
||||
seed: config.seed,
|
||||
},
|
||||
mutatedContract
|
||||
|
||||
@@ -57,7 +57,7 @@ async function expectLoadConfigError(
|
||||
test('schema: accepts minimal valid configs', () => {
|
||||
validateConfigAgainstSchema({}, CONFIG_SCHEMA);
|
||||
validateConfigAgainstSchema({ mode: 'verify', routes: ['GET /users'], seed: 42 }, CONFIG_SCHEMA);
|
||||
validateConfigAgainstSchema({ presets: { quick: { depth: 'quick', timeout: 5000 } } }, CONFIG_SCHEMA);
|
||||
validateConfigAgainstSchema({ presets: { quick: { runs: 10, timeout: 5000 } } }, CONFIG_SCHEMA);
|
||||
});
|
||||
|
||||
test('schema: rejects unknown keys with guidance', () => {
|
||||
@@ -131,11 +131,7 @@ test('schema: rejects enum and numeric range violations with clear guidance', ()
|
||||
path: 'profiles.default.mode',
|
||||
expectedGuidance: ['verify', 'observe', 'qualify'],
|
||||
},
|
||||
{
|
||||
value: { presets: { quick: { depth: 'super-deep' } } },
|
||||
path: 'presets.quick.depth',
|
||||
expectedGuidance: ['quick', 'standard', 'deep'],
|
||||
},
|
||||
|
||||
] as const;
|
||||
|
||||
for (const c of enumCases) {
|
||||
@@ -215,9 +211,9 @@ test('semantic: validates cross-reference and value rules', () => {
|
||||
{ value: { seed: 3.14 }, path: 'seed', guidance: 'integer' },
|
||||
{ value: { presets: { quick: { timeout: -100 } } }, path: 'presets.quick.timeout', guidance: 'non-negative' },
|
||||
{
|
||||
value: { presets: { quick: { depth: 'super-deep' } } },
|
||||
path: 'presets.quick.depth',
|
||||
guidance: 'quick, standard, deep',
|
||||
value: { presets: { quick: { timeout: -100 } } },
|
||||
path: 'presets.quick.timeout',
|
||||
guidance: 'non-negative',
|
||||
},
|
||||
] as const;
|
||||
|
||||
@@ -232,7 +228,7 @@ test('semantic: validates cross-reference and value rules', () => {
|
||||
{ routes: ['GET /users', 'POST /items'] },
|
||||
{ seed: -42 },
|
||||
{ seed: 0 },
|
||||
{ presets: { quick: { timeout: 0, depth: 'standard' } } },
|
||||
{ presets: { quick: { timeout: 0 } } },
|
||||
];
|
||||
|
||||
for (const value of acceptCases) {
|
||||
@@ -322,7 +318,7 @@ test('loadConfig: resolves profile and preset and applies profile overrides', as
|
||||
},
|
||||
presets: {
|
||||
quick: {
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
timeout: 5000,
|
||||
parallel: false,
|
||||
},
|
||||
|
||||
@@ -23,12 +23,10 @@
|
||||
* - Property and state model-based testing focused on confidence
|
||||
* - Iterative small steps with rapid feedback loops
|
||||
*/
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert';
|
||||
import { writeFileSync, readFileSync } from 'node:fs';
|
||||
import { resolve } from 'node:path';
|
||||
|
||||
import {
|
||||
migrateCommand,
|
||||
detectAllLegacyPatterns,
|
||||
@@ -36,31 +34,25 @@ import {
|
||||
type MigrateOptions,
|
||||
type MigrationItem,
|
||||
} from '../../cli/commands/migrate/index.js';
|
||||
|
||||
import {
|
||||
rewriteConfigFile,
|
||||
detectLegacyConfigFields,
|
||||
detectLegacyFieldsNoEquivalent,
|
||||
detectMixedLegacyModernFields,
|
||||
} from '../../cli/commands/migrate/rewriters/config-rewriter.js';
|
||||
|
||||
import {
|
||||
rewriteRouteAnnotations,
|
||||
detectLegacyRouteAnnotations,
|
||||
detectAmbiguousRoutePatterns,
|
||||
} from '../../cli/commands/migrate/rewriters/route-rewriter.js';
|
||||
|
||||
import {
|
||||
rewriteCodePatterns,
|
||||
detectLegacyCodePatterns,
|
||||
detectAmbiguousCodePatterns,
|
||||
} from '../../cli/commands/migrate/rewriters/code-rewriter.js';
|
||||
|
||||
import { createTempDir, cleanup, makeCtx } from './helpers.js';
|
||||
|
||||
test('migrate --check detects broad legacy config field set', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const legacyConfig = `export default {
|
||||
testMode: "verify",
|
||||
@@ -82,12 +74,9 @@ test('migrate --check detects broad legacy config field set', async () => {
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns are found');
|
||||
const legacyNames = result.items.map((item) => item.legacy);
|
||||
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
|
||||
@@ -103,29 +92,23 @@ test('migrate --check detects broad legacy config field set', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 1: Mixed legacy and modern config detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate detects mixed legacy and modern config fields', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
// Config with both legacy and modern fields present
|
||||
const mixedConfig = `export default {
|
||||
// Legacy field
|
||||
testMode: "verify",
|
||||
|
||||
// Modern field (conflicts with legacy)
|
||||
mode: "observe",
|
||||
|
||||
profiles: {
|
||||
quick: {
|
||||
preset: "safe-ci",
|
||||
},
|
||||
},
|
||||
|
||||
// Legacy container
|
||||
testProfiles: {
|
||||
old: {
|
||||
@@ -133,22 +116,17 @@ test('migrate detects mixed legacy and modern config fields', async () => {
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), mixedConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
// Should detect legacy patterns
|
||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
||||
assert.ok(result.items.length > 0, 'Should detect legacy items');
|
||||
|
||||
// Check that mixed fields are reported
|
||||
const legacyNames = result.items.map((item) => item.legacy);
|
||||
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode');
|
||||
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles');
|
||||
assert.ok(legacyNames.includes('usesPreset'), 'Should detect usesPreset');
|
||||
|
||||
// Verify guidance mentions the conflict
|
||||
const testModeItem = result.items.find((item) => item.legacy === 'testMode');
|
||||
assert.ok(testModeItem, 'Should have testMode item');
|
||||
@@ -157,19 +135,15 @@ test('migrate detects mixed legacy and modern config fields', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 2: Dry-run shows exact rewrites
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate dry-run shows exact file path, line number, legacy text, replacement text', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const legacyConfig = `export default {
|
||||
// Line 2
|
||||
testMode: "verify",
|
||||
|
||||
profiles: {
|
||||
quick: {
|
||||
// Line 7
|
||||
@@ -177,36 +151,27 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ dryRun: true }, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
||||
assert.ok(result.message, 'Should have output message');
|
||||
|
||||
// Verify dry-run output contains exact details
|
||||
assert.ok(result.message.includes('Dry run'), 'Should indicate dry run');
|
||||
assert.ok(result.message.includes('testMode'), 'Should show legacy text');
|
||||
assert.ok(result.message.includes('mode'), 'Should show replacement text');
|
||||
assert.ok(result.message.includes('usesPreset'), 'Should show usesPreset');
|
||||
assert.ok(result.message.includes('preset'), 'Should show preset replacement');
|
||||
|
||||
// Verify file path is shown
|
||||
assert.ok(result.message.includes('apophis.config.js'), 'Should show file path');
|
||||
|
||||
// Verify line numbers are shown
|
||||
assert.ok(result.message.includes(':2') || result.message.includes(': 2'), 'Should show line number');
|
||||
|
||||
// Verify total count
|
||||
assert.ok(result.message.includes('Total:'), 'Should show total count');
|
||||
assert.ok(result.message.includes('3'), 'Should show correct total (3 items)');
|
||||
|
||||
// Verify files would be modified
|
||||
assert.ok(result.filesWouldBeModified, 'Should list files that would be modified');
|
||||
assert.strictEqual(result.filesWouldBeModified.length, 1, 'Should show 1 file would be modified');
|
||||
|
||||
// Verify file was NOT modified
|
||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||
assert.ok(content.includes('testMode'), 'File should still have testMode');
|
||||
@@ -215,14 +180,11 @@ test('migrate dry-run shows exact file path, line number, legacy text, replaceme
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 3: Write performs rewrites correctly
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate write performs rewrites correctly', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const legacyConfig = `export default {
|
||||
testMode: "verify",
|
||||
@@ -232,16 +194,12 @@ test('migrate write performs rewrites correctly', async () => {
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ write: true }, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when rewrites performed');
|
||||
assert.ok(result.completed.length > 0, 'Should have completed items');
|
||||
assert.ok(result.filesModified && result.filesModified.length > 0, 'Should list modified files');
|
||||
|
||||
// Verify file WAS modified
|
||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||
assert.ok(!content.includes('testMode'), 'File should not have testMode');
|
||||
@@ -254,35 +212,26 @@ test('migrate write performs rewrites correctly', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 4: Ambiguous rewrite stops and shows context
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate ambiguous rewrite stops and shows surrounding context', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
// Create a file with an ambiguous code pattern
|
||||
const ambiguousCode = `import Fastify from 'fastify';
|
||||
const app = Fastify();
|
||||
|
||||
// This is ambiguous: what does oldApi() mean here?
|
||||
app.register(oldApi());
|
||||
|
||||
export default app;`;
|
||||
|
||||
writeFileSync(resolve(dir, 'app.js'), ambiguousCode);
|
||||
|
||||
// Also create a config file so migration has something to work with
|
||||
const config = `export default {
|
||||
mode: "verify",
|
||||
};`;
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), config);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ write: true }, ctx);
|
||||
|
||||
// Should stop with exit code 2 (USAGE_ERROR) because ambiguous patterns found
|
||||
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
|
||||
assert.ok(result.remaining.length > 0, 'Should have remaining items');
|
||||
@@ -290,21 +239,17 @@ export default app;`;
|
||||
assert.ok(result.message.includes('Ambiguous'), 'Should mention ambiguous patterns');
|
||||
assert.ok(result.message.includes('oldApi()'), 'Should show the ambiguous pattern');
|
||||
assert.ok(result.message.includes('manual choice'), 'Should mention manual choice');
|
||||
|
||||
// Verify context is shown (surrounding lines)
|
||||
assert.ok(result.message.includes('app.register'), 'Should show surrounding context');
|
||||
} finally {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 5: Legacy field with no equivalent emits guidance
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate legacy field with no direct equivalent emits human guidance', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
// Config with a legacy field that has no direct equivalent
|
||||
const legacyConfig = `export default {
|
||||
@@ -317,16 +262,12 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
|
||||
// This field is deprecated with no direct equivalent
|
||||
legacyField: true,
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
// Should detect the legacy field with no equivalent
|
||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 when legacy patterns found');
|
||||
assert.ok(result.items.length > 0, 'Should detect legacy items');
|
||||
|
||||
const legacyFieldItem = result.items.find((item) => item.legacy === 'legacyField');
|
||||
assert.ok(legacyFieldItem, 'Should detect legacyField');
|
||||
assert.ok(legacyFieldItem.guidance, 'Should have guidance for legacyField');
|
||||
@@ -343,14 +284,11 @@ test('migrate legacy field with no direct equivalent emits human guidance', asyn
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 6: Partial migration reports completed and remaining
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate partial migration reports completed and remaining items', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const legacyConfig = `export default {
|
||||
testMode: "verify",
|
||||
@@ -360,12 +298,9 @@ test('migrate partial migration reports completed and remaining items', async ()
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ write: true }, ctx);
|
||||
|
||||
assert.ok(result.completed.length > 0, 'Should have completed items');
|
||||
assert.ok(result.message, 'Should have output message');
|
||||
assert.ok(result.message.includes('Completed'), 'Should mention completed');
|
||||
@@ -374,20 +309,16 @@ test('migrate partial migration reports completed and remaining items', async ()
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 7: Preserves comments/formatting where feasible
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate preserves comments and formatting where feasible', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
// Config with specific formatting (comments, indentation)
|
||||
const legacyConfig = `export default {
|
||||
// This is a comment about testMode
|
||||
testMode: "verify",
|
||||
|
||||
/*
|
||||
* Block comment about testProfiles
|
||||
*/
|
||||
@@ -398,19 +329,14 @@ test('migrate preserves comments and formatting where feasible', async () => {
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ write: true }, ctx);
|
||||
|
||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||
|
||||
// Verify comments are preserved
|
||||
assert.ok(content.includes('// This is a comment about testMode'), 'Should preserve line comment');
|
||||
assert.ok(content.includes('Block comment about testProfiles'), 'Should preserve block comment');
|
||||
assert.ok(content.includes('// Inline comment'), 'Should preserve inline comment');
|
||||
|
||||
// Verify replacements were made
|
||||
assert.ok(content.includes('mode:'), 'Should have mode');
|
||||
assert.ok(content.includes('profiles:'), 'Should have profiles');
|
||||
@@ -419,14 +345,11 @@ test('migrate preserves comments and formatting where feasible', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 8: Migrate exits 0 when config is already modern
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate exits 0 when config is already modern', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const modernConfig = `export default {
|
||||
mode: "verify",
|
||||
@@ -438,7 +361,7 @@ test('migrate exits 0 when config is already modern', async () => {
|
||||
},
|
||||
presets: {
|
||||
"safe-ci": {
|
||||
depth: "quick",
|
||||
,
|
||||
timeout: 5000,
|
||||
},
|
||||
},
|
||||
@@ -448,12 +371,9 @@ test('migrate exits 0 when config is already modern', async () => {
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), modernConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 0, 'Should exit 0 for modern config');
|
||||
assert.strictEqual(result.items.length, 0, 'Should have no items');
|
||||
assert.ok(result.message, 'Should have message');
|
||||
@@ -462,35 +382,25 @@ test('migrate exits 0 when config is already modern', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 9: Migrate exits 2 when ambiguous in write mode
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate exits 2 when ambiguous patterns found in write mode', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const config = `export default {
|
||||
mode: "verify",
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), config);
|
||||
|
||||
// Create app with an ambiguous pattern
|
||||
const code = `import Fastify from 'fastify';
|
||||
const app = Fastify();
|
||||
|
||||
// Ambiguous pattern
|
||||
app.register(oldApi());
|
||||
|
||||
export default app;`;
|
||||
|
||||
writeFileSync(resolve(dir, 'app.js'), code);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ write: true }, ctx);
|
||||
|
||||
// Should exit 2 because ambiguous patterns found
|
||||
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when ambiguous patterns found in write mode');
|
||||
assert.ok(result.remaining.length > 0, 'Should have remaining ambiguous items');
|
||||
@@ -499,14 +409,11 @@ export default app;`;
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 10: Migrate emits guidance for each legacy field
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate emits guidance for each legacy field', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const legacyConfig = `export default {
|
||||
testMode: "verify",
|
||||
@@ -516,14 +423,10 @@ test('migrate emits guidance for each legacy field', async () => {
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
assert.ok(result.items.length > 0, 'Should have items');
|
||||
|
||||
for (const item of result.items) {
|
||||
assert.ok(item.guidance, `Item ${item.legacy} should have guidance`);
|
||||
assert.ok(
|
||||
@@ -535,14 +438,11 @@ test('migrate emits guidance for each legacy field', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 11: Config rewriter replaces legacy fields
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('config rewriter replaces legacy fields', () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const content = `export default {
|
||||
testMode: "verify",
|
||||
@@ -552,17 +452,13 @@ test('config rewriter replaces legacy fields', () => {
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'test.config.js'), content);
|
||||
|
||||
const items = detectLegacyConfigFields(content, 'test.config.js');
|
||||
assert.strictEqual(items.length, 3, 'Should detect 3 legacy fields');
|
||||
|
||||
const result = rewriteConfigFile(
|
||||
resolve(dir, 'test.config.js'),
|
||||
items,
|
||||
);
|
||||
|
||||
assert.strictEqual(result.modified, true, 'Should modify content');
|
||||
assert.ok(result.content.includes('mode:'), 'Should have mode');
|
||||
assert.ok(result.content.includes('profiles:'), 'Should have profiles');
|
||||
@@ -572,35 +468,28 @@ test('config rewriter replaces legacy fields', () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 12: Route rewriter detects x-validate-runtime annotation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('route rewriter detects x-validate-runtime annotation', () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const content = `export default {
|
||||
schema: {
|
||||
'x-validate-runtime': true,
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'test.routes.js'), content);
|
||||
|
||||
const items = detectLegacyRouteAnnotations(content, 'test.routes.js');
|
||||
assert.strictEqual(items.length, 1, 'Should detect 1 legacy annotation');
|
||||
const firstItem = items[0];
|
||||
assert.ok(firstItem, 'Expected one migration item');
|
||||
assert.strictEqual(firstItem.legacy, 'x-validate-runtime');
|
||||
assert.strictEqual(firstItem.replacement, 'runtime');
|
||||
|
||||
const result = rewriteRouteAnnotations(
|
||||
resolve(dir, 'test.routes.js'),
|
||||
items,
|
||||
);
|
||||
|
||||
assert.strictEqual(result.modified, true, 'Should modify content');
|
||||
assert.ok(result.content.includes("'runtime'"), 'Should have runtime');
|
||||
assert.ok(!result.content.includes('x-validate-runtime'), 'Should not have legacy annotation');
|
||||
@@ -608,34 +497,25 @@ test('route rewriter detects x-validate-runtime annotation', () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 13: Code rewriter detects legacy patterns
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('code rewriter detects legacy patterns', () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const content = `import Fastify from 'fastify';
|
||||
const app = Fastify();
|
||||
|
||||
app.register(contract());
|
||||
app.register(stateful());
|
||||
app.register(scenario());
|
||||
|
||||
export default app;`;
|
||||
|
||||
writeFileSync(resolve(dir, 'test.app.js'), content);
|
||||
|
||||
const items = detectLegacyCodePatterns(content, 'test.app.js');
|
||||
assert.strictEqual(items.length, 3, 'Should detect 3 legacy patterns');
|
||||
|
||||
const result = rewriteCodePatterns(
|
||||
resolve(dir, 'test.app.js'),
|
||||
items,
|
||||
);
|
||||
|
||||
assert.strictEqual(result.modified, true, 'Should modify content');
|
||||
assert.ok(result.content.includes("verify({ kind: 'contract' })"), 'Should have verify');
|
||||
assert.ok(result.content.includes("qualify({ kind: 'stateful' })"), 'Should have qualify stateful');
|
||||
@@ -644,28 +524,21 @@ export default app;`;
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 14: Dry-run default mode (safe by default)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate defaults to dry-run mode (safe by default)', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const legacyConfig = `export default {
|
||||
testMode: "verify",
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
// No mode specified — should default to dry-run
|
||||
const result = await migrateCommand({}, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1 in dry-run mode');
|
||||
assert.ok(result.message?.includes('Dry run'), 'Should indicate dry run');
|
||||
|
||||
// Verify file was NOT modified
|
||||
const content = readFileSync(resolve(dir, 'apophis.config.js'), 'utf-8');
|
||||
assert.ok(content.includes('testMode'), 'File should still have testMode');
|
||||
@@ -673,38 +546,30 @@ test('migrate defaults to dry-run mode (safe by default)', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 15: Mixed legacy/modern field detection at rewriter level
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('config rewriter detects mixed legacy and modern fields', () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const content = `export default {
|
||||
// Both legacy and modern present
|
||||
testMode: "verify",
|
||||
mode: "observe",
|
||||
|
||||
testProfiles: {
|
||||
quick: {
|
||||
usesPreset: "safe-ci",
|
||||
},
|
||||
},
|
||||
|
||||
profiles: {
|
||||
modern: {
|
||||
preset: "safe-ci",
|
||||
},
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'test.config.js'), content);
|
||||
|
||||
const mixedReports = detectMixedLegacyModernFields(content, 'test.config.js');
|
||||
assert.ok(mixedReports.length > 0, 'Should detect mixed fields');
|
||||
|
||||
const testModeReport = mixedReports.find((r) => r.legacy === 'testMode');
|
||||
assert.ok(testModeReport, 'Should report testMode as mixed');
|
||||
assert.ok(testModeReport.guidance.includes('testMode'), 'Guidance should mention testMode');
|
||||
@@ -713,14 +578,11 @@ test('config rewriter detects mixed legacy and modern fields', () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 16: Ambiguous route pattern detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('route rewriter detects ambiguous route patterns with context', () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const content = `export default {
|
||||
schema: {
|
||||
@@ -728,12 +590,9 @@ test('route rewriter detects ambiguous route patterns with context', () => {
|
||||
'x-validate': true,
|
||||
},
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'test.routes.js'), content);
|
||||
|
||||
const items = detectAmbiguousRoutePatterns(content, 'test.routes.js');
|
||||
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
|
||||
|
||||
const firstItem = items[0];
|
||||
assert.ok(firstItem, 'Expected one migration item');
|
||||
assert.strictEqual(firstItem.legacy, 'x-validate');
|
||||
@@ -744,28 +603,20 @@ test('route rewriter detects ambiguous route patterns with context', () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 17: Ambiguous code pattern detection with context
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('code rewriter detects ambiguous code patterns with surrounding context', () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const content = `import Fastify from 'fastify';
|
||||
const app = Fastify();
|
||||
|
||||
// Ambiguous pattern
|
||||
app.register(oldApi());
|
||||
|
||||
export default app;`;
|
||||
|
||||
writeFileSync(resolve(dir, 'test.app.js'), content);
|
||||
|
||||
const items = detectAmbiguousCodePatterns(content, 'test.app.js');
|
||||
assert.strictEqual(items.length, 1, 'Should detect 1 ambiguous pattern');
|
||||
|
||||
const firstItem = items[0];
|
||||
assert.ok(firstItem, 'Expected one migration item');
|
||||
assert.strictEqual(firstItem.legacy, 'oldApi()');
|
||||
@@ -777,42 +628,32 @@ export default app;`;
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 18: Legacy fixture detection
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate detects legacy patterns in fixture config', async () => {
|
||||
const ctx = makeCtx({ cwd: 'src/cli/__fixtures__/legacy-config' });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 1, 'Should detect legacy patterns in fixture');
|
||||
assert.ok(result.items.length > 0, 'Should find legacy items');
|
||||
|
||||
const legacyNames = result.items.map((item) => item.legacy);
|
||||
assert.ok(legacyNames.includes('testMode'), 'Should detect testMode in fixture');
|
||||
assert.ok(legacyNames.includes('testProfiles'), 'Should detect testProfiles in fixture');
|
||||
assert.ok(legacyNames.includes('testPresets'), 'Should detect testPresets in fixture');
|
||||
assert.ok(legacyNames.includes('envPolicies'), 'Should detect envPolicies in fixture');
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 19: JSON output format
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate outputs JSON format with all fields', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const legacyConfig = `export default {
|
||||
testMode: "verify",
|
||||
};`;
|
||||
|
||||
writeFileSync(resolve(dir, 'apophis.config.js'), legacyConfig);
|
||||
|
||||
const ctx = makeCtx({ cwd: dir, options: { ...makeCtx().options, format: 'json' } });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 1, 'Should exit 1');
|
||||
assert.ok(result.items.length > 0, 'Should have items');
|
||||
assert.ok(result.totalRewrites, 'Should have totalRewrites');
|
||||
@@ -821,18 +662,14 @@ test('migrate outputs JSON format with all fields', async () => {
|
||||
cleanup(dir);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test 20: No files found returns usage error
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('migrate returns usage error when no files found', async () => {
|
||||
const dir = createTempDir();
|
||||
|
||||
try {
|
||||
const ctx = makeCtx({ cwd: dir });
|
||||
const result = await migrateCommand({ check: true }, ctx);
|
||||
|
||||
assert.strictEqual(result.exitCode, 2, 'Should exit 2 when no files found');
|
||||
assert.ok(result.message?.includes('No config or app files found'), 'Should mention no files found');
|
||||
} finally {
|
||||
|
||||
@@ -152,7 +152,7 @@ test('contract runner supports cross-operation APOSTL ensures', async () => {
|
||||
await fastify.register(apophisPlugin, {})
|
||||
registerItemApi(fastify)
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({ depth: 'quick', seed: 7 })
|
||||
const result = await fastify.apophis.contract({ runs: 10, seed: 7 })
|
||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||
assert.strictEqual(failures.length, 0)
|
||||
} finally {
|
||||
@@ -169,7 +169,7 @@ test('stateful runner supports cross-operation APOSTL ensures', async () => {
|
||||
await fastify.register(apophisPlugin, {})
|
||||
registerItemApi(fastify)
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 11 })
|
||||
const result = await fastify.apophis.stateful({ runs: 10, seed: 11 })
|
||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||
assert.strictEqual(failures.length, 0)
|
||||
} finally {
|
||||
@@ -217,7 +217,7 @@ test('contract runner applies chaos injection when configured', async () => {
|
||||
}, async () => ({ ok: true }))
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
seed: 3,
|
||||
chaos: {
|
||||
probability: 1,
|
||||
@@ -241,7 +241,7 @@ test('contract runner supports previous(...) with response_body(this) path place
|
||||
await fastify.register(apophisPlugin, {})
|
||||
registerPlanApi(fastify)
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({ depth: 'quick', seed: 17 })
|
||||
const result = await fastify.apophis.contract({ runs: 10, seed: 17 })
|
||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||
assert.strictEqual(failures.length, 0)
|
||||
} finally {
|
||||
@@ -258,7 +258,7 @@ test('stateful runner supports previous(...) with response_body(this) path place
|
||||
await fastify.register(apophisPlugin, {})
|
||||
registerPlanApi(fastify)
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 19 })
|
||||
const result = await fastify.apophis.stateful({ runs: 10, seed: 19 })
|
||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||
assert.strictEqual(failures.length, 0)
|
||||
} finally {
|
||||
@@ -406,7 +406,7 @@ test('contract runner validates hypermedia links with route_exists', async () =>
|
||||
})
|
||||
registerHypermediaApi(fastify)
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({ depth: 'quick', seed: 23 })
|
||||
const result = await fastify.apophis.contract({ runs: 10, seed: 23 })
|
||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||
if (failures.length > 0) {
|
||||
console.log('Contract failures:', failures.map((f: TestResult) => ({
|
||||
@@ -441,7 +441,7 @@ test('stateful runner validates cross-route relationships', async () => {
|
||||
})
|
||||
registerHypermediaApi(fastify)
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.stateful({ depth: 'quick', seed: 29 })
|
||||
const result = await fastify.apophis.stateful({ runs: 10, seed: 29 })
|
||||
const failures = result.tests.filter((entry: TestResult) => !entry.ok)
|
||||
if (failures.length > 0) {
|
||||
console.log('Stateful failures:', failures.map((f: TestResult) => ({
|
||||
|
||||
@@ -36,7 +36,7 @@ test('APOPHIS_DEBUG=1 logs requests and responses', async () => {
|
||||
originalDebug(msg, _obj)
|
||||
}
|
||||
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(result.tests.length > 0, 'should have tests')
|
||||
|
||||
// Should have logged at least one request and one response
|
||||
@@ -80,7 +80,7 @@ test('APOPHIS_DEBUG=0 does not log requests', async () => {
|
||||
originalDebug(msg, _obj)
|
||||
}
|
||||
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(result.tests.length > 0, 'should have tests')
|
||||
|
||||
// Should not have any request/response logs
|
||||
|
||||
@@ -30,7 +30,7 @@ test('example: minimal API compiles and runs', async () => {
|
||||
|
||||
await fastify.ready()
|
||||
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(result.tests.length > 0, 'should have test results')
|
||||
console.log('Minimal example:', result.summary)
|
||||
} finally {
|
||||
@@ -111,7 +111,7 @@ test('example: CRUD API with contracts compiles and runs', async () => {
|
||||
|
||||
await fastify.ready()
|
||||
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(result.tests.length > 0, 'should have test results')
|
||||
console.log('CRUD example:', result.summary)
|
||||
} finally {
|
||||
@@ -149,7 +149,7 @@ test('example: prefix registration works', async () => {
|
||||
const itemContract = contracts.find(c => c.path === '/api/v1/items')
|
||||
assert.ok(itemContract, 'should discover prefixed route')
|
||||
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(result.tests.length > 0, 'should run tests on prefixed routes')
|
||||
} finally {
|
||||
await fastify.close()
|
||||
|
||||
@@ -243,7 +243,7 @@ test('petit-runner executes tests against real API', async () => {
|
||||
]
|
||||
const fastifyWithRoutes = Object.assign(fastify, { routes: mockRoutes })
|
||||
const result = await runPetitTests(fastifyWithRoutes as any, {
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: undefined
|
||||
})
|
||||
@@ -367,7 +367,7 @@ test('full integration: plugin + routes + test execution', async () => {
|
||||
const createUserContract = contracts.find(c => c.path === '/users' && c.method === 'POST')
|
||||
assert.ok(createUserContract, 'create user contract should exist')
|
||||
assert.strictEqual(createUserContract.category, 'constructor')
|
||||
const testResult = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const testResult = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(Array.isArray(testResult.tests), 'tests should be an array')
|
||||
assert.ok(testResult.tests.length > 0, 'tests should not be empty')
|
||||
await fastify.apophis.cleanup()
|
||||
@@ -439,7 +439,7 @@ test('mode filtering: stateful mode only runs constructor/mutator routes', async
|
||||
}, async () => ({ status: 'ok' }))
|
||||
await fastify.ready()
|
||||
// Run in stateful mode
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
// In stateful mode, utility routes should be excluded
|
||||
// The test should only run constructor and mutator routes
|
||||
assert.ok(Array.isArray(result.tests), 'tests should be an array')
|
||||
@@ -474,7 +474,7 @@ test('failing contract produces ContractViolation with suggestion', async () =>
|
||||
return { status: 'created' } // Returns 200, not 201
|
||||
})
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
// Find the failing test
|
||||
const failingTests = result.tests.filter(t => !t.ok)
|
||||
assert.ok(failingTests.length > 0, 'should have at least one failing test')
|
||||
@@ -647,7 +647,7 @@ test('integration: contract routes option limits tested routes', async () => {
|
||||
}, async () => ({ ok: true }))
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
routes: ['GET /included'],
|
||||
})
|
||||
const includedTests = result.tests.filter(t => t.name.includes('GET /included'))
|
||||
@@ -673,7 +673,7 @@ test('integration: contract variants are tagged and run in declared order', asyn
|
||||
}, async () => ({ ok: true }))
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
variants: [
|
||||
{ name: 'json', headers: { accept: 'application/json' } },
|
||||
{ name: 'xml', headers: { accept: 'application/xml' } },
|
||||
@@ -715,7 +715,7 @@ test('integration: variant headers override scope headers', async () => {
|
||||
}, async () => ({ ok: true }))
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
variants: [
|
||||
{ name: 'xml', headers: { accept: 'application/xml' } },
|
||||
],
|
||||
@@ -744,7 +744,7 @@ test('integration: route-level x-variants are extracted and executed', async ()
|
||||
}, async () => ({ ok: true }))
|
||||
await fastify.ready()
|
||||
// No call-site variants; route-level variants should drive execution
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
const jsonTests = result.tests.filter((t) => t.name.includes('[variant:json]'))
|
||||
const xmlTests = result.tests.filter((t) => t.name.includes('[variant:xml]'))
|
||||
assert.ok(jsonTests.length > 0, 'route json variant should produce tests')
|
||||
@@ -781,7 +781,7 @@ test('integration: inferred contracts are guarded by status code', async () => {
|
||||
return { error: 'not found' }
|
||||
})
|
||||
await fastify.ready()
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
// Should pass because the inferred const contract is guarded:
|
||||
// response_code(this) == 200 => response_body(this).status == "success"
|
||||
// The 404 response doesn't trigger the antecedent, so the implication holds.
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
import { convertSchema } from '../domain/schema-to-arbitrary.js'
|
||||
import { lookupCache, storeCache } from '../incremental/cache.js'
|
||||
import type { ApiCommand } from '../domain/stateful.js'
|
||||
import type { DepthConfig, RouteContract } from '../types.js'
|
||||
import type { RouteContract, RunConfig } from '../types.js'
|
||||
import * as fc from 'fast-check'
|
||||
|
||||
const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
|
||||
@@ -19,11 +19,10 @@ const buildCommand = (route: RouteContract, data: unknown): ApiCommand => ({
|
||||
|
||||
export const generateCommands = (
|
||||
routes: RouteContract[],
|
||||
depth: DepthConfig,
|
||||
runConfig: RunConfig,
|
||||
seed?: number,
|
||||
generationProfile: 'quick' | 'standard' | 'thorough' = 'standard',
|
||||
): { commands: ApiCommand[][], cacheHits: number, cacheMisses: number } => {
|
||||
const commandsPerRoute = Math.max(1, Math.floor(depth.contractRuns / Math.max(routes.length, 1)))
|
||||
const commandsPerRoute = Math.max(1, Math.floor(runConfig.contractRuns / Math.max(routes.length, 1)))
|
||||
let cacheHits = 0
|
||||
let cacheMisses = 0
|
||||
const allCommands = routes.map((route) => {
|
||||
@@ -40,7 +39,7 @@ export const generateCommands = (
|
||||
cacheMisses++
|
||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||
const bodyArb = bodySchema !== undefined
|
||||
? convertSchema(bodySchema, { context: 'request', generationProfile })
|
||||
? convertSchema(bodySchema, { context: 'request' })
|
||||
: fc.constant({})
|
||||
const pathParams = route.path.match(/:([a-zA-Z_][a-zA-Z0-9_]*)/g) ?? []
|
||||
const pathParamArbs: Record<string, fc.Arbitrary<string>> = {}
|
||||
@@ -82,7 +81,7 @@ import { buildPetitSuite, filterPetitRoutes } from './route-filter.js'
|
||||
import { executePetitCommandStep } from './petit-command-step.js'
|
||||
import { runTripleBoundaryPropertyTest } from './triple-boundary-runner.js'
|
||||
import { makeTrackedResource } from '../domain/state-operations.js'
|
||||
import { resolveDepth, resolveGenerationProfile } from '../types.js'
|
||||
import { resolveRuns } from '../types.js'
|
||||
import type {
|
||||
EvalContext,
|
||||
FastifyInjectInstance,
|
||||
@@ -133,9 +132,8 @@ export const runPetitTests = async (
|
||||
}
|
||||
}
|
||||
|
||||
const depth = resolveDepth(config.depth ?? 'standard')
|
||||
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
|
||||
const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, depth, config.seed, generationProfile)
|
||||
const runConfig = resolveRuns(config.runs)
|
||||
const { commands: commandGroups, cacheHits, cacheMisses } = generateCommands(routes, runConfig, config.seed)
|
||||
const allCommands = commandGroups.flat()
|
||||
const rng = config.seed !== undefined ? new SeededRng(config.seed) : undefined
|
||||
|
||||
@@ -181,7 +179,6 @@ export const runPetitTests = async (
|
||||
? createOutboundMockRuntime({
|
||||
contracts: outboundContractRegistry.resolve(Array.from(outboundNames)),
|
||||
mode: config.outboundMocks?.mode ?? 'example',
|
||||
generationProfile,
|
||||
overrides: config.outboundMocks?.overrides,
|
||||
unmatched: config.outboundMocks?.unmatched ?? 'error',
|
||||
seed: config.seed !== undefined ? hashCombine(config.seed, 0x6d6f636b) : Math.floor(Math.random() * 0xffffffff),
|
||||
|
||||
@@ -7,7 +7,7 @@ import apophisPlugin from '../index.js'
|
||||
import type { TestResult } from '../types.js'
|
||||
type TestFastifyInstance = FastifyInstance & {
|
||||
apophis: {
|
||||
contract: (opts?: { depth?: string; scope?: string; seed?: number }) => Promise<any>
|
||||
contract: (opts?: { runs?: number; scope?: string; seed?: number }) => Promise<any>
|
||||
spec: () => Record<string, unknown>
|
||||
}
|
||||
}
|
||||
@@ -44,19 +44,19 @@ test('scope isolation: routes with x-scope are filtered by scope parameter', asy
|
||||
}, async () => ({ user: true }))
|
||||
await fastify.ready()
|
||||
// Test with no scope - should discover all 3 routes
|
||||
const allResult = await fastify.apophis.contract({ depth: 'quick', scope: undefined })
|
||||
const allResult = await fastify.apophis.contract({ runs: 10, scope: undefined })
|
||||
const allPaths = new Set(allResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
||||
assert.ok(allPaths.has('/public'), 'public route should be in all scope')
|
||||
assert.ok(allPaths.has('/admin'), 'admin route should be in all scope')
|
||||
assert.ok(allPaths.has('/user'), 'user route should be in all scope')
|
||||
// Test with admin scope - should only get public + admin
|
||||
const adminResult = await fastify.apophis.contract({ depth: 'quick', scope: 'admin' })
|
||||
const adminResult = await fastify.apophis.contract({ runs: 10, scope: 'admin' })
|
||||
const adminPaths = new Set(adminResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
||||
assert.ok(adminPaths.has('/public'), 'public route should be in admin scope')
|
||||
assert.ok(adminPaths.has('/admin'), 'admin route should be in admin scope')
|
||||
assert.ok(!adminPaths.has('/user'), 'user route should NOT be in admin scope')
|
||||
// Test with user scope - should only get public + user
|
||||
const userResult = await fastify.apophis.contract({ depth: 'quick', scope: 'user' })
|
||||
const userResult = await fastify.apophis.contract({ runs: 10, scope: 'user' })
|
||||
const userPaths = new Set(userResult.tests.map((t: TestResult) => t.name.split(' ')[1]))
|
||||
assert.ok(userPaths.has('/public'), 'public route should be in user scope')
|
||||
assert.ok(!userPaths.has('/admin'), 'admin route should NOT be in user scope')
|
||||
@@ -88,7 +88,7 @@ test('scope isolation: scope headers are passed to requests', async () => {
|
||||
headers: { 'x-custom-header': 'test-value' },
|
||||
metadata: {}
|
||||
})
|
||||
await fastify.apophis.contract({ depth: 'quick', scope: 'test' })
|
||||
await fastify.apophis.contract({ runs: 10, scope: 'test' })
|
||||
assert.strictEqual(receivedHeaders['x-custom-header'], 'test-value', 'scope header should be passed to request')
|
||||
} finally {
|
||||
await fastify.close()
|
||||
@@ -130,7 +130,7 @@ test('scope isolation: non-matching scope returns empty test suite', async () =>
|
||||
}, async () => ({ ok: true }))
|
||||
await fastify.ready()
|
||||
// Test with non-matching scope
|
||||
const result = await fastify.apophis.contract({ depth: 'quick', scope: 'other' })
|
||||
const result = await fastify.apophis.contract({ runs: 10, scope: 'other' })
|
||||
assert.strictEqual(result.tests.length, 0, 'no tests should run for non-matching scope')
|
||||
assert.strictEqual(result.summary.passed, 0, 'no tests should pass')
|
||||
assert.strictEqual(result.summary.failed, 0, 'no tests should fail')
|
||||
|
||||
@@ -32,7 +32,7 @@ test('serverless: fastify.ready() without listen works', async () => {
|
||||
await fastify.ready()
|
||||
|
||||
// Should be able to run tests
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(result.tests.length > 0, 'should have tests')
|
||||
|
||||
// Should be able to get spec
|
||||
@@ -115,7 +115,7 @@ test('serverless: multiple ready() calls are safe', async () => {
|
||||
// Second ready() should be safe (idempotent)
|
||||
await fastify.ready()
|
||||
|
||||
const result = await fastify.apophis.contract({ depth: 'quick' })
|
||||
const result = await fastify.apophis.contract({ runs: 10 })
|
||||
assert.ok(result.tests.length > 0, 'should still work after multiple ready() calls')
|
||||
} finally {
|
||||
await fastify.close()
|
||||
|
||||
@@ -22,7 +22,7 @@ test('stateful runner handles empty routes', async () => {
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
})
|
||||
@@ -62,7 +62,7 @@ test('stateful runner executes commands', async () => {
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
})
|
||||
@@ -103,7 +103,7 @@ test('stateful runner detects status code violations', async () => {
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
})
|
||||
@@ -139,7 +139,7 @@ test('stateful runner evaluates APOSTL formulas', async () => {
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
})
|
||||
@@ -189,7 +189,7 @@ test('stateful runner tracks resource state', async () => {
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
})
|
||||
@@ -264,7 +264,7 @@ test('stateful runner substitutes path params from resource state', async () =>
|
||||
await fastify.ready()
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
})
|
||||
@@ -308,7 +308,7 @@ test('stateful runner supports config-level variants', async () => {
|
||||
await fastify.ready()
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
variants: [
|
||||
@@ -359,7 +359,7 @@ test('stateful runner supports route-level x-variants', async () => {
|
||||
await fastify.ready()
|
||||
|
||||
const result = await runStatefulTests(fastify as any, {
|
||||
depth: 'quick',
|
||||
runs: 10,
|
||||
scope: undefined,
|
||||
seed: 42,
|
||||
})
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
* generate → execute → validate → update → check-invariants
|
||||
*/
|
||||
import type { ExtensionRegistry } from '../extension/types.js'
|
||||
import { resolveDepth, resolveGenerationProfile } from '../types.js'
|
||||
import { resolveRuns } from '../types.js'
|
||||
import { discoverRoutes } from '../domain/discovery.js'
|
||||
import { convertSchema } from '../domain/schema-to-arbitrary.js'
|
||||
import { SeededRng } from '../infrastructure/seeded-rng.js'
|
||||
@@ -22,7 +22,7 @@ import type { OutboundContractRegistry } from '../domain/outbound-contracts.js'
|
||||
import * as fc from 'fast-check'
|
||||
import type { ModelState } from '../domain/stateful.js'
|
||||
import type { CleanupManager } from '../infrastructure/cleanup-manager.js'
|
||||
import type { DepthConfig, EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js'
|
||||
import type { EvalContext, FastifyInjectInstance, RouteContract, ScopeRegistry, TestConfig, TestResult, TestSuite } from '../types.js'
|
||||
|
||||
// Pure: hash helpers for deterministic sub-seeds
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -61,7 +61,6 @@ class ApiOperation implements StatefulApiOperation {
|
||||
// ============================================================================
|
||||
const createCommandArbitrary = (
|
||||
routes: RouteContract[],
|
||||
generationProfile: 'quick' | 'standard' | 'thorough',
|
||||
): { arb: fc.Arbitrary<ApiOperation>, cacheHits: number, cacheMisses: number } => {
|
||||
let cacheHits = 0
|
||||
let cacheMisses = 0
|
||||
@@ -74,7 +73,7 @@ const createCommandArbitrary = (
|
||||
cacheMisses++
|
||||
const bodySchema = route.schema?.body as Record<string, unknown> | undefined
|
||||
const arb = bodySchema !== undefined
|
||||
? convertSchema(bodySchema, { context: 'request', generationProfile })
|
||||
? convertSchema(bodySchema, { context: 'request' })
|
||||
: fc.constant({})
|
||||
return arb.map((params) => new ApiOperation(route, params as Record<string, unknown>))
|
||||
})
|
||||
@@ -93,8 +92,7 @@ export const runStatefulTests = async (
|
||||
outboundContractRegistry?: OutboundContractRegistry
|
||||
): Promise<TestSuite> => {
|
||||
const startTime = Date.now()
|
||||
const depth = resolveDepth(config.depth ?? 'standard')
|
||||
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
|
||||
const runConfig = resolveRuns(config.runs)
|
||||
if (extensionRegistry) {
|
||||
await extensionRegistry.runSuiteStartHooks(config)
|
||||
}
|
||||
@@ -151,7 +149,7 @@ export const runStatefulTests = async (
|
||||
variantName ? `[variant:${variantName}] ${name}` : name
|
||||
// Get scope headers for test requests
|
||||
const baseScopeHeaders = scopeRegistry?.getHeaders(config.scope ?? null) ?? {}
|
||||
const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes, generationProfile)
|
||||
const { arb: commandArb, cacheHits, cacheMisses } = createCommandArbitrary(routes)
|
||||
let allResults: TestResult[] = []
|
||||
let globalTestId = 0
|
||||
// Create seeded RNG for reproducible path param selection
|
||||
@@ -178,7 +176,6 @@ export const runStatefulTests = async (
|
||||
suiteMockRuntime = createOutboundMockRuntime({
|
||||
contracts: allResolved,
|
||||
mode: config.outboundMocks?.mode ?? 'example',
|
||||
generationProfile,
|
||||
overrides: config.outboundMocks?.overrides,
|
||||
unmatched: config.outboundMocks?.unmatched ?? 'error',
|
||||
seed: outboundSeed,
|
||||
@@ -186,7 +183,7 @@ export const runStatefulTests = async (
|
||||
suiteMockRuntime.install()
|
||||
}
|
||||
// Run property-based stateful tests per variant
|
||||
const numRuns = depth.statefulRuns
|
||||
const numRuns = runConfig.statefulRuns
|
||||
const seed = config.seed
|
||||
let counterexampleOutput: string | undefined
|
||||
const hashString = (s: string): number => {
|
||||
@@ -276,7 +273,7 @@ export const runStatefulTests = async (
|
||||
}
|
||||
try {
|
||||
const prop = fc.asyncProperty(
|
||||
fc.array(commandArb, { minLength: 1, maxLength: depth.maxCommands }),
|
||||
fc.array(commandArb, { minLength: 1, maxLength: runConfig.maxCommands }),
|
||||
async (cmds) => {
|
||||
await runSequence(cmds)
|
||||
return true
|
||||
|
||||
@@ -22,7 +22,7 @@ import type {
|
||||
TestConfig,
|
||||
TestResult,
|
||||
} from '../types.js'
|
||||
import { resolveDepth, resolveGenerationProfile } from '../types.js'
|
||||
import { resolveRuns } from '../types.js'
|
||||
|
||||
export const runTripleBoundaryPropertyTest = async (
|
||||
route: RouteContract,
|
||||
@@ -39,10 +39,9 @@ export const runTripleBoundaryPropertyTest = async (
|
||||
if (!config.chaos) return []
|
||||
|
||||
const results: TestResult[] = []
|
||||
const generationProfile = config.generationProfile ?? resolveGenerationProfile(config.depth)
|
||||
const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos, generationProfile)
|
||||
const depth = resolveDepth(config.depth ?? 'standard')
|
||||
const numRuns = Math.max(10, Math.floor(depth.contractRuns / 2))
|
||||
const arbitrary = createTripleBoundaryArbitrary(route, contracts, config.chaos)
|
||||
const runConfig = resolveRuns(config.runs)
|
||||
const numRuns = Math.max(10, Math.floor(runConfig.contractRuns / 2))
|
||||
|
||||
const property = fc.asyncProperty(arbitrary, async (cmd) => {
|
||||
const testId = testIdBase + results.length + 1
|
||||
|
||||
+2
-5
@@ -39,18 +39,15 @@ export type {
|
||||
|
||||
// Formula and test configuration types
|
||||
export type {
|
||||
TestDepth,
|
||||
TestConfig,
|
||||
ResolvedOutboundContract,
|
||||
OutboundChaosConfig,
|
||||
ChaosConfig,
|
||||
DepthConfig,
|
||||
RunConfig,
|
||||
} from './types/formula.js'
|
||||
|
||||
export {
|
||||
DEPTH_CONFIGS,
|
||||
resolveDepth,
|
||||
resolveGenerationProfile,
|
||||
resolveRuns,
|
||||
} from './types/formula.js'
|
||||
|
||||
// Extension types
|
||||
|
||||
+1
-1
@@ -201,7 +201,7 @@ export interface ApophisTestDecorations {
|
||||
|
||||
// Forward declarations to avoid circular deps — these are defined in sibling modules
|
||||
export interface TestConfig {
|
||||
readonly depth?: import('./formula.js').TestDepth
|
||||
readonly runs?: number
|
||||
readonly scope?: string
|
||||
readonly seed?: number
|
||||
readonly timeout?: number
|
||||
|
||||
+10
-34
@@ -7,11 +7,8 @@
|
||||
// Test: Configuration
|
||||
// ============================================================================
|
||||
|
||||
export type TestDepth = 'quick' | 'standard' | 'thorough' | { runs: number }
|
||||
|
||||
export interface TestConfig {
|
||||
readonly depth?: TestDepth
|
||||
readonly generationProfile?: GenerationProfile
|
||||
readonly runs?: number
|
||||
readonly scope?: string
|
||||
readonly seed?: number
|
||||
readonly timeout?: number
|
||||
@@ -204,49 +201,28 @@ export interface ChaosConfig {
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Depth Configuration
|
||||
// Test run configuration
|
||||
// ============================================================================
|
||||
|
||||
export interface DepthConfig {
|
||||
export interface RunConfig {
|
||||
readonly contractRuns: number
|
||||
readonly propertyRuns: number
|
||||
readonly statefulRuns: number
|
||||
readonly maxCommands: number
|
||||
}
|
||||
|
||||
export type GenerationProfile = 'quick' | 'standard' | 'thorough'
|
||||
const DEFAULT_RUNS = 50
|
||||
|
||||
export const DEPTH_CONFIGS: Record<'quick' | 'standard' | 'thorough', DepthConfig> = {
|
||||
quick: { contractRuns: 10, propertyRuns: 50, statefulRuns: 5, maxCommands: 10 },
|
||||
standard: { contractRuns: 50, propertyRuns: 100, statefulRuns: 20, maxCommands: 30 },
|
||||
thorough: { contractRuns: 200, propertyRuns: 1000, statefulRuns: 100, maxCommands: 50 }
|
||||
}
|
||||
|
||||
export function resolveDepth(depth: TestDepth): DepthConfig {
|
||||
if (typeof depth === 'string') {
|
||||
return DEPTH_CONFIGS[depth]
|
||||
}
|
||||
export function resolveRuns(runs: number | undefined): RunConfig {
|
||||
const r = runs ?? DEFAULT_RUNS
|
||||
return {
|
||||
contractRuns: depth.runs,
|
||||
propertyRuns: depth.runs,
|
||||
statefulRuns: Math.max(1, Math.floor(depth.runs / 10)),
|
||||
maxCommands: Math.max(5, Math.floor(depth.runs / 5)),
|
||||
contractRuns: r,
|
||||
propertyRuns: r * 2,
|
||||
statefulRuns: Math.max(1, Math.floor(r / 10)),
|
||||
maxCommands: Math.max(5, Math.floor(r / 2)),
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveGenerationProfile(depth: TestDepth | undefined): GenerationProfile {
|
||||
if (depth === undefined) {
|
||||
return 'standard'
|
||||
}
|
||||
if (typeof depth === 'string') {
|
||||
return depth
|
||||
}
|
||||
|
||||
if (depth.runs <= 25) return 'quick'
|
||||
if (depth.runs >= 250) return 'thorough'
|
||||
return 'standard'
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Results
|
||||
// ============================================================================
|
||||
|
||||
Reference in New Issue
Block a user