feat: add acceptance tests for comparison [CMPA-595]

Author: tonidevine1

test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Package.swift

@@ -0,0 +1,12 @@
+// swift-tools-version:5.6
+import PackageDescription
+
+let package = Package(
+    name: "LocalDep",
+    products: [
+        .library(name: "LocalDep", targets: ["LocalDep"]),
+    ],
+    targets: [
+        .target(name: "LocalDep"),
+    ]
+)

test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Sources/LocalDep/LocalDep.swift

@@ -0,0 +1,3 @@
+public enum LocalDep {
+    public static let greeting = "hello"
+}

test/acceptance/workspaces/swift-local-dep/Package.swift

@@ -0,0 +1,21 @@
+// swift-tools-version:5.6
+// SwiftPM fixture for the unified-test-api equivalence suite.
+//
+// Unlike most lockfile ecosystems, the snyk-swiftpm-plugin resolves by shelling
+// out to `swift package show-dependencies`, which needs the dependencies on disk.
+// To stay network-free and deterministic we depend on a sibling package by path
+// (Deps/LocalDep) instead of a remote git URL — no fetch, no vendored checkout,
+// only the `swift` toolchain itself (gated via requiresCmd in the spec).
+import PackageDescription
+
+let package = Package(
+    name: "App",
+    dependencies: [
+        .package(path: "Deps/LocalDep"),
+    ],
+    targets: [
+        .executableTarget(name: "App", dependencies: [
+            .product(name: "LocalDep", package: "LocalDep"),
+        ]),
+    ]
+)

test/acceptance/workspaces/swift-local-dep/Sources/App/main.swift

@@ -0,0 +1,3 @@
+import LocalDep
+
+print(LocalDep.greeting)

test/jest/acceptance/snyk-test/equivalenceHelpers.ts

@@ -64,6 +64,14 @@ export async function runBothFlows(
   server: FakeServer,
   env: Record<string, string | undefined>,
   runOptions: RunCommandOptions = {},
+  /**
+   * Extra feature flags to enable ONLY in the unified (FF-on) run — e.g. a
+   * native resolver gate like 'internal_new_gradle_resolver'. The legacy run
+   * never sees these, so the comparison becomes "native resolver vs legacy
+   * CLI". Empty (the default) keeps both runs on the legacy resolver, which is
+   * the Phase 1 endpoint-parity comparison.
+   */
+  unifiedFlags: string[] = [],
 ): Promise<EquivalenceResult> {
   const argsWithJson = argsString.includes('--json')
     ? argsString
@@ -83,6 +91,9 @@ export async function runBothFlows(
 
   server.restore();
   server.setFeatureFlag(UNIFIED_TEST_API_FF, true);
+  for (const flag of unifiedFlags) {
+    server.setFeatureFlag(flag, true);
+  }
   const unifiedRun = await runSnykCLI(argsWithJson, {
     ...runOptions,
     cwd,
@@ -162,6 +173,11 @@ export type AssertOptions = {
   /** Set true for fixtures that legitimately produce no submissions
    *  (e.g. "no supported target files"). */
   expectNoSubmissions?: boolean;
+  /** Set true for fixtures expected to be REJECTED by both flows (e.g. an
+   *  out-of-sync lockfile). The two flows have intentional exit-code
+   *  differences for failures (TS CLI exits 3, os-flows exits 2), so this
+   *  asserts the tolerance invariant "both fail" rather than equal codes. */
+  expectError?: boolean;
 };
 
 export function assertEquivalent(
@@ -170,6 +186,23 @@ export function assertEquivalent(
 ): EquivalenceDiff {
   const { legacy, unified } = result;
 
+  // Error-parity mode: the fixture should be rejected by BOTH flows. Exit-code
+  // values legitimately differ between the TS CLI and os-flows, so we assert
+  // only the invariant that neither flow reported success.
+  if (options.expectError) {
+    if (legacy.code !== 0 && unified.code !== 0) {
+      return { ok: true };
+    }
+    return {
+      ok: false,
+      reason: `expected both flows to fail: legacy=${legacy.code} unified=${unified.code}`,
+      detail: {
+        legacyStderr: legacy.stderr,
+        unifiedStderr: unified.stderr,
+      },
+    };
+  }
+
   const bothEmpty =
     legacy.submissionCount === 0 && unified.submissionCount === 0;
 

test/jest/acceptance/snyk-test/unified-test-api-equivalence.spec.ts

@@ -10,16 +10,44 @@
  * FF on  → Go binary's os-flows extension resolves dep graphs via the plugin
  *           orchestrator and posts to /rest/orgs/:orgId/tests.
  *
- * Starter corpus. Expand per resolveDepgraphs-rollout-plan.md.
+ * Phase 1 of the rollout: breadth across every ecosystem that resolves through
+ * the legacy-CLI fallback. The native resolver flags (cargo, pnpm, gradle, …)
+ * stay OFF here, so BOTH runs resolve via the same legacy CLI and only the
+ * submission endpoint differs — that endpoint/serialization parity is what this
+ * phase proves. Native-resolver parity (gradle, pnpm) and exit-1/error parity
+ * arrive in later phases.
+ *
+ * Fixtures that need an external build tool to resolve (go, sbt, swift, …) are
+ * tagged with `requiresCmd` and skip when that tool is absent rather than
+ * failing "inconclusive". Pure lockfile ecosystems (yarn, ruby, composer,
+ * poetry, …) need no tool beyond what the harness already has.
+ *
+ * Note: pip/pipenv and hex/mix are deliberately excluded — their snyk plugins
+ * resolve by introspecting an *installed* environment (site-packages / fetched
+ * mix deps), so they need an install step and can't resolve from a clean
+ * checkout. Python is covered via Poetry, which resolves offline from its lock.
  */
 
+import { execFileSync } from 'child_process';
 import { fakeServer } from '../../../acceptance/fake-server';
 import { createProjectFromWorkspace } from '../../util/createProject';
 import { getServerPort } from '../../util/getServerPort';
 import { assertEquivalent, runBothFlows } from './equivalenceHelpers';
 
 jest.setTimeout(1000 * 60 * 3);
 
+/** True when `cmd` resolves on PATH — used to skip fixtures whose toolchain is absent. */
+function commandAvailable(cmd: string): boolean {
+  try {
+    execFileSync(process.platform === 'win32' ? 'where' : 'which', [cmd], {
+      stdio: 'ignore',
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 describe('snyk test — unified test API equivalence (FF off vs on)', () => {
   let server;
   let env: Record<string, string>;
@@ -47,9 +75,15 @@ describe('snyk test — unified test API equivalence (FF off vs on)', () => {
     name: string;
     args: string;
     expectNoSubmissions?: boolean;
+    /** Expected to be rejected by both flows (e.g. out-of-sync lockfile). */
+    expectError?: boolean;
+    /** Binary that must be on PATH for the legacy CLI to resolve this fixture.
+     *  Omitted for lockfile-only ecosystems that need no external build tool. */
+    requiresCmd?: string;
   };
 
   const fixtures: Fixture[] = [
+    // --- Starter corpus (unchanged) ---
     { name: 'npm-package', args: 'test' },
     { name: 'maven-app', args: 'test' },
     { name: 'mono-repo-project', args: 'test --all-projects' },
@@ -58,24 +92,81 @@ describe('snyk test — unified test API equivalence (FF off vs on)', () => {
       args: 'test',
       expectNoSubmissions: true,
     },
+
+    // --- Phase 1: breadth via the legacy fallback ---
+    // JavaScript (yarn) — resolved from yarn.lock; no tool beyond node.
+    { name: 'yarn-package', args: 'test' },
+    { name: 'yarn-workspaces', args: 'test --all-projects' },
+    // Ruby — resolved from Gemfile.lock.
+    { name: 'ruby-app', args: 'test' },
+    // PHP (Composer) — resolved from composer.lock.
+    { name: 'composer-app', args: 'test' },
+    // CocoaPods — resolved from Podfile.lock.
+    { name: 'cocoapods-app', args: 'test' },
+    // Swift (SwiftPM) — the plugin shells out to `swift package show-dependencies`,
+    // so it needs the swift toolchain (gated) and the deps on disk. The fixture
+    // depends on a sibling package by path, keeping resolution network-free.
+    { name: 'swift-local-dep', args: 'test', requiresCmd: 'swift' },
+    // Go modules — needs the go toolchain.
+    { name: 'golang-gomodules', args: 'test', requiresCmd: 'go' },
+    // Python (Poetry) — resolved from poetry.lock; no tool beyond node.
+    // (pip/pipenv are intentionally NOT used here: their snyk plugins resolve by
+    // introspecting *installed* site-packages, so they require a pip/pipenv
+    // install step and can't resolve from a clean checkout the way a lockfile can.)
+    { name: 'poetry-app', args: 'test' },
+    // .NET (NuGet) — resolved from project.assets.json; no tool beyond node.
+    { name: 'nuget-app-2', args: 'test' },
+    // Scala (sbt) — needs sbt.
+    { name: 'sbt-app', args: 'test', requiresCmd: 'sbt' },
+
+    // --- Phase 2: option parity (same flag applied to BOTH flows) ---
+    // Each option is passed to legacy and unified alike; the resolved graph and
+    // project metadata must still match. Clean projects, so exit code stays 0.
+    { name: 'npm-package', args: 'test --dev' },
+    { name: 'npm-package', args: 'test --file=package.json' },
+    {
+      name: 'npm-package-pruneable',
+      args: 'test --prune-repeated-subdependencies',
+    },
+
+    // --- Phase 2: partial-failure parity under --all-projects ---
+    // monorepo-bad-project mixes resolvable and unresolvable projects. This is
+    // the exact scenario the os-flows change targeted ("tolerate per-project
+    // resolution failures in the --all-projects orchestrator path"): both flows
+    // should resolve the good projects, skip the bad one, and agree on the
+    // submitted-project count and exit code.
+    { name: 'monorepo-bad-project', args: 'test --all-projects' },
+
+    // --- Phase 2: error parity (both flows must REJECT the bad input) ---
+    // Exit-code values legitimately differ (TS CLI 3 vs os-flows 2), so these
+    // assert the tolerance invariant "both fail" rather than equal codes.
+    { name: 'npm-out-of-sync', args: 'test', expectError: true },
+    { name: 'yarn-out-of-sync', args: 'test', expectError: true },
   ];
 
   describe.each(fixtures)(
     'fixture: $name ($args)',
-    ({ name, args, expectNoSubmissions }) => {
-      test('FF off vs on produces equivalent dep graphs and exit code', async () => {
-        const project = await createProjectFromWorkspace(name);
+    ({ name, args, expectNoSubmissions, expectError, requiresCmd }) => {
+      const runnable = !requiresCmd || commandAvailable(requiresCmd);
+      (runnable ? test : test.skip)(
+        'FF off vs on produces equivalent dep graphs and exit code',
+        async () => {
+          const project = await createProjectFromWorkspace(name);
 
-        const result = await runBothFlows(project.path(), args, server, env);
-        const diff = assertEquivalent(result, { expectNoSubmissions });
+          const result = await runBothFlows(project.path(), args, server, env);
+          const diff = assertEquivalent(result, {
+            expectNoSubmissions,
+            expectError,
+          });
 
-        if (!diff.ok) {
-          throw new Error(
-            `Equivalence failed for ${name} (${args}): ${diff.reason}\n` +
-              `detail=${JSON.stringify(diff.detail, null, 2)}`,
-          );
-        }
-      });
+          if (!diff.ok) {
+            throw new Error(
+              `Equivalence failed for ${name} (${args}): ${diff.reason}\n` +
+                `detail=${JSON.stringify(diff.detail, null, 2)}`,
+            );
+          }
+        },
+      );
     },
   );
 });