Merge pull request #340 from coderoad/feature/subtasks

Feature/subtasks

Author: ShMcK

Diff for: CHANGELOG.md

@@ -125,3 +125,33 @@ CODEROAD_TUTORIAL_URL='path/to/tutorial_config_file.json' // will load directly
 ## [0.6.1]
 
 - Replace checkboxes with icons
+
+## [0.7.0]
+
+- Support loading subtasks (#340). Subtasks are a list of tests that need to pass before a task is complete. They can be loaded by:
+
+1. filtering down to a subset of tests by setting the `step.setup.filter` to a regex pattern that matches the tests you're targeting
+2. setting the `step.setup.subtasks` variable to true
+
+- Change for the test runner config. Changes are backwards compatible.
+
+1. `testRunner.path`=> `testRunner.directory`
+2. `testRunner.actions` => `testRunner.setup`
+3. Change command to capture `args` for "TAP" support, and test "filter"ing support. These changes will help lead to specific test suite presets in the future.
+
+```json
+{
+  "testRunner": {
+    "command": "mocha",
+    "args": {
+      "filter": "--grep",
+      "tap": "--reporter=mocha-tap-reporter"
+    },
+    "directory": ".coderoad",
+    "setup": {
+      "commits": ["410bd4f"],
+      "commands": ["npm install"]
+    }
+  }
+}
+```

Diff for: package.json

@@ -45,7 +45,7 @@
     "@typescript-eslint/parser": "^2.33.0",
     "chokidar": "^3.4.0",
     "dotenv": "^8.2.0",
-    "eslint": "^7.0.0",
+    "eslint": "^6.8.0",
     "eslint-config-prettier": "^6.11.0",
     "eslint-plugin-prettier": "^3.1.3",
     "git-url-parse": "^11.1.2",

Diff for: src/actions/setupActions.ts

@@ -10,10 +10,10 @@ import logger from '../services/logger'
 interface SetupActions {
   actions: TT.StepActions
   send: (action: T.Action) => void // send messages to client
-  path?: string
+  dir?: string
 }
 
-export const setupActions = async ({ actions, send, path }: SetupActions): Promise<void> => {
+export const setupActions = async ({ actions, send, dir }: SetupActions): Promise<void> => {
   if (!actions) {
     return
   }
@@ -45,7 +45,7 @@ export const setupActions = async ({ actions, send, path }: SetupActions): Promi
 
   // 4. run command
   if (!alreadyLoaded) {
-    await runCommands({ commands: commands || [], send, path }).catch(onError)
+    await runCommands({ commands: commands || [], send, dir }).catch(onError)
   }
 }
 

Diff for: src/actions/tutorialConfig.ts

@@ -6,12 +6,12 @@ import * as git from '../services/git'
 import { DISABLE_RUN_ON_SAVE } from '../environment'
 
 interface TutorialConfigParams {
-  config: TT.TutorialConfig
+  data: TT.Tutorial
   alreadyConfigured?: boolean
   onComplete?(): void
 }
 
-const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage | void> => {
+const tutorialConfig = async ({ data, alreadyConfigured }: TutorialConfigParams): Promise<E.ErrorMessage | void> => {
   if (!alreadyConfigured) {
     // setup git, add remote
     const initError: E.ErrorMessage | void = await git.initIfNotExists().catch(
@@ -27,7 +27,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
     }
 
     // verify that internet is connected, remote exists and branch exists
-    const remoteConnectError: E.ErrorMessage | void = await git.checkRemoteConnects(config.repo).catch(
+    const remoteConnectError: E.ErrorMessage | void = await git.checkRemoteConnects(data.config.repo).catch(
       (error: Error): E.ErrorMessage => ({
         type: 'FailedToConnectToGitRepo',
         message: error.message,
@@ -40,7 +40,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
     }
 
     // TODO if remote not already set
-    const coderoadRemoteError: E.ErrorMessage | void = await git.setupCodeRoadRemote(config.repo.uri).catch(
+    const coderoadRemoteError: E.ErrorMessage | void = await git.setupCodeRoadRemote(data.config.repo.uri).catch(
       (error: Error): E.ErrorMessage => ({
         type: 'GitRemoteAlreadyExists',
         message: error.message,
@@ -52,7 +52,7 @@ const tutorialConfig = async ({ config, alreadyConfigured }: TutorialConfigParam
     }
   }
 
-  await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER, config.testRunner)
+  await vscode.commands.executeCommand(COMMANDS.CONFIG_TEST_RUNNER, data)
 
   if (!DISABLE_RUN_ON_SAVE) {
     // verify if file test should run based on document saved

Diff for: src/actions/utils/loadWatchers.ts

@@ -38,9 +38,11 @@ const loadWatchers = (watchers: string[]) => {
         const now = +new Date()
         if (!lastFire || lastFire - now > 1000) {
           vscode.commands.executeCommand(COMMANDS.RUN_TEST, {
-            onSuccess: () => {
-              // cleanup watcher on success
-              disposeWatcher(watcher)
+            callbacks: {
+              onSuccess: () => {
+                // cleanup watcher on success
+                disposeWatcher(watcher)
+              },
             },
           })
         }

Diff for: src/actions/utils/runCommands.ts

@@ -4,10 +4,10 @@ import { exec } from '../../services/node'
 interface RunCommands {
   commands: string[]
   send: (action: T.Action) => void
-  path?: string
+  dir?: string
 }
 
-const runCommands = async ({ commands, send, path }: RunCommands) => {
+const runCommands = async ({ commands, send, dir }: RunCommands) => {
   if (!commands.length) {
     return
   }
@@ -19,7 +19,7 @@ const runCommands = async ({ commands, send, path }: RunCommands) => {
     send({ type: 'COMMAND_START', payload: { process: { ...process, status: 'RUNNING' } } })
     let result: { stdout: string; stderr: string }
     try {
-      result = await exec({ command, path })
+      result = await exec({ command, dir })
       console.log(result)
     } catch (error) {
       console.log(`Test failed: ${error.message}`)

Diff for: src/channel/index.ts

@@ -204,7 +204,7 @@ class Channel implements Channel {
             }
           }
 
-          const error: E.ErrorMessage | void = await tutorialConfig({ config: data.config }).catch((error: Error) => ({
+          const error: E.ErrorMessage | void = await tutorialConfig({ data }).catch((error: Error) => ({
             type: 'UnknownError',
             message: `Location: tutorial config.\n\n${error.message}`,
           }))
@@ -231,9 +231,8 @@ class Channel implements Channel {
           if (!tutorialContinue) {
             throw new Error('Invalid tutorial to continue')
           }
-          const continueConfig: TT.TutorialConfig = tutorialContinue.config
           await tutorialConfig({
-            config: continueConfig,
+            data: tutorialContinue,
             alreadyConfigured: true,
           })
           // update the current stepId on startup
@@ -307,7 +306,7 @@ class Channel implements Channel {
         await vscode.commands.executeCommand(COMMANDS.SET_CURRENT_POSITION, action.payload.position)
         await solutionActions({ actions: action.payload.actions, send: this.send })
         // run test following solution to update position
-        vscode.commands.executeCommand(COMMANDS.RUN_TEST)
+        vscode.commands.executeCommand(COMMANDS.RUN_TEST, { subtasks: true })
         return
       case 'EDITOR_SYNC_PROGRESS':
         // update progress when a level is deemed complete in the client
@@ -318,7 +317,7 @@ class Channel implements Channel {
         await showOutput(channel)
         return
       case 'EDITOR_RUN_TEST':
-        vscode.commands.executeCommand(COMMANDS.RUN_TEST)
+        vscode.commands.executeCommand(COMMANDS.RUN_TEST, action?.payload)
         return
       default:
         logger(`No match for action type: ${actionType}`)

Diff for: src/editor/commands.ts

@@ -50,13 +50,19 @@ export const createCommands = ({ extensionPath, workspaceState }: CreateCommandP
       // setup 1x1 horizontal layout
       webview.createOrShow()
     },
-    [COMMANDS.CONFIG_TEST_RUNNER]: async (config: TT.TutorialTestRunnerConfig) => {
-      if (config.actions) {
+    [COMMANDS.CONFIG_TEST_RUNNER]: async (data: TT.Tutorial) => {
+      const testRunnerConfig = data.config.testRunner
+      const setup = testRunnerConfig.setup || testRunnerConfig.actions // TODO: deprecate and remove config.actions
+      if (setup) {
         // setup tutorial test runner commits
         // assumes git already exists
-        await setupActions({ actions: config.actions, send: webview.send, path: config.path })
+        await setupActions({
+          actions: setup,
+          send: webview.send,
+          dir: testRunnerConfig.directory || testRunnerConfig.path,
+        }) // TODO: deprecate and remove config.path
       }
-      testRunner = createTestRunner(config, {
+      testRunner = createTestRunner(data, {
         onSuccess: (position: T.Position) => {
           logger('test pass position', position)
           // send test pass message back to client
@@ -75,20 +81,27 @@ export const createCommands = ({ extensionPath, workspaceState }: CreateCommandP
           // send test run message back to client
           webview.send({ type: 'TEST_RUNNING', payload: { position } })
         },
+        onLoadSubtasks: ({ summary }) => {
+          webview.send({ type: 'LOAD_TEST_SUBTASKS', payload: { summary } })
+        },
       })
     },
     [COMMANDS.SET_CURRENT_POSITION]: (position: T.Position) => {
       // set from last setup stepAction
       currentPosition = position
     },
-    [COMMANDS.RUN_TEST]: (callback?: { onSuccess: () => void }) => {
+    [COMMANDS.RUN_TEST]: ({
+      subtasks,
+      callbacks,
+    }: { subtasks?: boolean; callbacks?: { onSuccess: () => void } } = {}) => {
       logger('run test current', currentPosition)
       // use stepId from client, or last set stepId
       // const position: T.Position = {
       //   ...current,
       //   stepId: current && current.position.stepId?.length ? current.position.stepId : currentPosition.stepId,
       // }
-      testRunner(currentPosition, callback?.onSuccess)
+      logger('currentPosition', currentPosition)
+      testRunner({ position: currentPosition, onSuccess: callbacks?.onSuccess, subtasks })
     },
   }
 }

Diff for: src/services/node/index.ts

@@ -8,11 +8,11 @@ const asyncExec = promisify(cpExec)
 
 interface ExecParams {
   command: string
-  path?: string
+  dir?: string
 }
 
 export const exec = (params: ExecParams): Promise<{ stdout: string; stderr: string }> | never => {
-  const cwd = join(WORKSPACE_ROOT, params.path || '')
+  const cwd = join(WORKSPACE_ROOT, params.dir || '')
   return asyncExec(params.command, { cwd })
 }
 

Diff for: src/services/testRunner/index.ts

@@ -2,7 +2,7 @@ import * as T from 'typings'
 import * as TT from 'typings/tutorial'
 import { exec } from '../node'
 import logger from '../logger'
-import parser from './parser'
+import parser, { ParserOutput } from './parser'
 import { debounce, throttle } from './throttle'
 import onError from '../sentry/onError'
 import { clearOutput, addOutput } from './output'
@@ -13,14 +13,22 @@ interface Callbacks {
   onFail(position: T.Position, failSummary: T.TestFail): void
   onRun(position: T.Position): void
   onError(position: T.Position): void
+  onLoadSubtasks({ summary }: { summary: { [testName: string]: boolean } }): void
 }
 
 const failChannelName = 'CodeRoad (Tests)'
 const logChannelName = 'CodeRoad (Logs)'
 
-const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callbacks) => {
-  return async (position: T.Position, onSuccess?: () => void): Promise<void> => {
-    logger('createTestRunner', position)
+interface TestRunnerParams {
+  position: T.Position
+  subtasks?: boolean
+  onSuccess?: () => void
+}
+
+const createTestRunner = (data: TT.Tutorial, callbacks: Callbacks) => {
+  const testRunnerConfig = data.config.testRunner
+  const testRunnerFilterArg = testRunnerConfig.args?.filter
+  return async ({ position, onSuccess, subtasks }: TestRunnerParams): Promise<void> => {
     const startTime = throttle()
     // throttle time early
     if (!startTime) {
@@ -30,11 +38,35 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
     logger('------------------- RUN TEST -------------------')
 
     // flag as running
-    callbacks.onRun(position)
+    if (!subtasks) {
+      callbacks.onRun(position)
+    }
 
     let result: { stdout: string | undefined; stderr: string | undefined }
     try {
-      result = await exec({ command: config.command, path: config.path })
+      let command = testRunnerConfig.args
+        ? `${testRunnerConfig.command} ${testRunnerConfig?.args.tap}`
+        : testRunnerConfig.command // TODO: enforce TAP
+
+      // filter tests if requested
+      if (testRunnerFilterArg) {
+        // get tutorial step from position
+        // check the step actions for specific command
+        // NOTE: cannot just pass in step actions as the test can be called by:
+        // - onEditorSave, onWatcher, onSolution, onRunTest, onSubTask
+        const levels = data.levels
+        const level = levels.find((l) => l.id === position.levelId)
+        const step = level?.steps.find((s) => s.id === position.stepId)
+        const testFilter = step?.setup?.filter
+        if (testFilter) {
+          // append filter commands
+          command = [command, testRunnerFilterArg, testFilter].join(' ')
+        } else {
+          throw new Error('Test Runner filter not configured')
+        }
+      }
+      logger('COMMAND', command)
+      result = await exec({ command, dir: testRunnerConfig.directory || testRunnerConfig.path }) // TODO: remove config.path later
     } catch (err) {
       result = { stdout: err.stdout, stderr: err.stack }
     }
@@ -49,7 +81,13 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
 
     const { stdout, stderr } = result
 
-    const tap = parser(stdout || '')
+    const tap: ParserOutput = parser(stdout || '')
+
+    if (subtasks) {
+      callbacks.onLoadSubtasks({ summary: tap.summary })
+      // exit early
+      return
+    }
 
     addOutput({ channel: logChannelName, text: tap.logs.join('\n'), show: false })
 
@@ -60,6 +98,7 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
         const failSummary = {
           title: firstFail.message || 'Test Failed',
           description: firstFail.details || 'Unknown error',
+          summary: tap.summary,
         }
         callbacks.onFail(position, failSummary)
         const output = formatFailOutput(tap)
@@ -76,7 +115,9 @@ const createTestRunner = (config: TT.TutorialTestRunnerConfig, callbacks: Callba
     // PASS
     if (tap.ok) {
       clearOutput(failChannelName)
+
       callbacks.onSuccess(position)
+
       if (onSuccess) {
         onSuccess()
       }

Diff for: src/services/testRunner/parser.test.ts

@@ -6,7 +6,13 @@ describe('parser', () => {
 1..1
 ok 1 - Should pass
 `
-    expect(parser(example)).toEqual({ ok: true, passed: [{ message: 'Should pass' }], failed: [], logs: [] })
+    expect(parser(example)).toEqual({
+      ok: true,
+      passed: [{ message: 'Should pass' }],
+      failed: [],
+      logs: [],
+      summary: { 'Should pass': true },
+    })
   })
   test('should detect multiple successes', () => {
     const example = `
@@ -20,6 +26,10 @@ ok 2 - Should also pass
       passed: [{ message: 'Should pass' }, { message: 'Should also pass' }],
       failed: [],
       logs: [],
+      summary: {
+        'Should pass': true,
+        'Should also pass': true,
+      },
     })
   })
   test('should detect failure if no tests passed', () => {
@@ -170,6 +180,10 @@ at processImmediate (internal/timers.js:439:21)`,
         },
       ],
       logs: ['log 1', 'log 2'],
+      summary: {
+        'package.json should have "express" installed': true,
+        'server should log "Hello World"': false,
+      },
     })
   })
 })