Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 33 additions & 12 deletions .drone.star
Original file line number Diff line number Diff line change
Expand Up @@ -586,14 +586,14 @@ def e2eTestsOnPlaywright(ctx):

steps += ocisService()

steps += [{
steps.append({
"name": "e2e-tests-playwright",
"image": OC_CI_NODEJS_IMAGE,
"environment": environment,
"commands": [
"pnpm test:e2e:playwright --project=chromium",
],
}]
})

if not "skip-a11y" in ctx.build.title.lower():
steps += uploadA11yResult(ctx)
Expand Down Expand Up @@ -691,6 +691,16 @@ def e2eTests(ctx):
"SKIP_A11Y_TESTS": params["skipA11y"],
}

if "suites" in matrix:
environment["TEST_SUITES"] = ",".join(params["suites"])
steps += filterTestSuitesToRun(ctx, params["suites"])
elif "features" in matrix:
environment["FEATURE_FILES"] = ",".join(params["features"])
steps += filterTestSuitesToRun(ctx, params["features"])
else:
print("Error: No suites or features defined for e2e test suite '%s'" % suite)
return []

steps += restoreBuildArtifactCache(ctx, "pnpm", ".pnpm-store") + \
installPnpm() + \
restoreBrowsersCache() + \
Expand Down Expand Up @@ -720,22 +730,15 @@ def e2eTests(ctx):
steps += (tikaService() if params["tikaNeeded"] else []) + \
ocisService(params["extraServerEnvironment"])

command = "bash run-e2e.sh "
if "suites" in matrix:
command += "--suites %s" % ",".join(params["suites"])
elif "features" in matrix:
command += "%s" % " ".join(params["features"])
else:
print("Error: No suites or features defined for e2e test suite '%s'" % suite)
return []

steps += [{
"name": "e2e-tests",
"image": OC_CI_NODEJS_IMAGE,
"environment": environment,
"commands": [
"cat %s/tests/drone/suites.env" % dir["web"],
". %s/tests/drone/suites.env || true" % dir["web"],
"cd tests/e2e",
command,
"bash run-e2e.sh",
],
}] + \
uploadTracingResult(ctx) + \
Expand Down Expand Up @@ -2046,3 +2049,21 @@ def uploadA11yResult(ctx):
},
},
]

def filterTestSuitesToRun(ctx, suites = []):
if "full-ci" in ctx.build.title.lower() and ctx.build.event == "cron":
return []
if len(suites) and "cucumber/" in suites[0]:
ENV = "FEATURE_FILES="
else:
ENV = "TEST_SUITES="
return [
{
"name": "filter-suites-to-run",
"image": OC_CI_NODEJS_IMAGE,
"commands": [
"node %s/tests/drone/filterTestSuitesToRun.js %s" % (dir["web"], ",".join(suites)),
"cat %s/tests/drone/suites.env" % dir["web"],
],
},
]
50 changes: 36 additions & 14 deletions docs/testing/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ $ cd web
$ pnpm install
```

### Unit Tests
## Unit Tests

We have a steadily growing coverage of unit tests. You can run them locally via

Expand All @@ -34,7 +34,7 @@ $ pnpm -r test:unit
You can also specify which package to run the test on, such as: `pnpm --filter @ownclouders/web-pkg test:unit`.
Alternatively, tests can also be run by navigating to the package name and then running `pnpm test:unit`.

#### Unit Test File Structure
### Unit Test File Structure

Our unit tests spec files follow a simple structure:

Expand All @@ -46,16 +46,16 @@ We usually organize tests with nested `describe` blocks. If you would like to ge
the structure, scope and goals of your unit tests before actually writing some, we invite you to make a pull request
with only `describe` blocks and nested `it.todo("put your test description here")` lines.

### E2E Tests (Playwright)
## E2E Tests (Playwright)

Our end-to-end test suite is built upon the [Playwright Framework](https://github.com/microsoft/playwright),
which makes it easy to write tests, debug them and have them run cross-browser with minimal overhead.

#### Preparation
### Preparation

Please make sure you have installed all dependencies and started the server(s) as described in [tooling]({{< ref "tooling.md#development-setup" >}}).

#### Prepare Web
### Prepare Web

Bundle the web frontend with the following command:

Expand All @@ -65,7 +65,7 @@ $ pnpm build

Our compose setup automatically mounts it into an oCIS backend, respectively. Web also gets recompiled on changes.

#### Start Web
### Start Web

Start the web with the following command:

Expand All @@ -75,15 +75,15 @@ docker compose up

This will start all the services. The ENV variables specific to each services are defined in the `docker-compose.yml` file.

#### Run E2E Tests
### Run E2E Tests

The following command will run all available e2e tests:

```shell
$ pnpm test:e2e:cucumber 'tests/e2e/cucumber/**/*.feature'
```

#### Options
### Options

To run a particular test, simply add the feature file and line number to the test command, e.g. `pnpm test:e2e:cucumber tests/e2e/cucumber/features/smoke/admin-settings/users.feature:84`

Expand Down Expand Up @@ -111,7 +111,7 @@ To then open e.g. the tracing from the `REPORT_DIR`, run
$ npx playwright show-trace path/to/file.zip
```

#### Lint E2E Test Code
### Lint E2E Test Code

Run the following command to find out the lint issues early in the test codes:

Expand All @@ -127,7 +127,7 @@ $ pnpm lint --fix

If the lint problems are not fixed by `--fix` option, we have to manually fix the code.

### Analyze the Test Report
## Analyze the Test Report

The cucumber library is used as the test runner for e2e tests. The report generator script lives inside the `tests/e2e/cucumber/report` folder. If you want to create a report after the tests are done, run the command:

Expand All @@ -144,15 +144,15 @@ To see all available options run
node tests/e2e/cucumber/report --help
```

### E2E Tests on oCIS With Keycloak
## E2E Tests on oCIS With Keycloak

We can run some of the e2e tests on oCIS setup with Keycloak as an external idp. To run tests against locally, please follow the steps below:

#### Run oCIS With Keycloak
### Run oCIS With Keycloak

There's a documentation to serve [oCIS with Keycloak](https://owncloud.dev/ocis/deployment/ocis_keycloak/). Please follow each step to run **oCIS with Keycloak**.

#### Run E2E Tests
### Run E2E Tests

```bash
KEYCLOAK=true \
Expand All @@ -167,7 +167,7 @@ Following environment variables come in use while running e2e tests on oCIS with
- `KEYCLOAK=true` runs the tests with Keycloak
- `KEYCLOAK_REALM` sets oCIS realm name used on Keycloak

### E2E Tests With Predefiend Users (`@predefined-users`)
## E2E Tests With Predefiend Users (`@predefined-users`)

It is possible to run e2e tests with predefined users. This is useful for running tests in a production-like environment.
The following environment variables are used to run the tests with predefined users:
Expand Down Expand Up @@ -242,3 +242,25 @@ All tests which are related to:
- Features enabled/disabled
- Running latest tests against an older version of oCIS/Web
- Large file uploads may take longer time

## Usage of `web-packages.txt` In the Test Suite

Test suites may include the `web-packages.txt` file to denote which web packages changes affect the defined test scenarios. This information is used in CI pipelines to determine which test suites to run based on the changed web packages.

The `web-packages.txt` file should be included within the test suite directory as shown below:

```
└── tests/e2e/cucumber/features
└── admin-settings
├── users.feature
└── web-packages.txt
```

And the `web-packages.txt` file should list the dependent web packages, one per line, for example:

NOTE: The package name should start with `web-` in order to be recognized correctly, if not, the line will be ignored.

```
web-app-files
web-app-admin-settings
```
165 changes: 165 additions & 0 deletions tests/drone/filterTestSuitesToRun.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
import { execSync } from 'child_process'
import fs from 'fs'
import path from 'path'

const targetBranch = process.env.DRONE_TARGET_BRANCH || 'master'

// paths that if changed will run all test suites
const mandatoryPaths = ['tests/e2e/', 'tests/drone/', '.drone.star', '.drone.env', 'package.json']

// INFO: 1 and 2 elements are node and script name respectively
const scriptDir = path.dirname(process.argv[1])
const suitesToCheck = process.argv[2]
.split(',')
.map((suite) => suite.trim())
.filter((suite) => suite)

// list of test suites with dependent packages
// Example:
// {
// 'web-app-ocm': ['ocm'],
// 'web-app-search': ['search']
// }
const packageToTestSuiteMap = {}

/*
--------------------
web packages
--------------------
*/
const excludePackages = ['web-container', 'web-test-helpers']
// default packages that affect all test suites
const defaultDependentPackages = ['design-system', 'web-client', 'web-runtime', 'web-pkg']
const packagesDir = `${scriptDir}/../../packages`
const allWebPackages = fs
.readdirSync(packagesDir)
.filter(
(entry) =>
entry.startsWith('web-') &&
!defaultDependentPackages.includes(entry) &&
fs.statSync(path.join(packagesDir, entry)).isDirectory() &&
!excludePackages.includes(entry)
)

/*
--------------------
test suites
--------------------
*/
const testSuitesDir = `${scriptDir}/../e2e/cucumber/features`
const testSuites = fs.readdirSync(testSuitesDir).filter((entry) => {
if (!fs.statSync(path.join(testSuitesDir, entry)).isDirectory()) {
return false
}
const webPackagesFile = path.join(testSuitesDir, entry, 'web-packages.txt')
if (fs.existsSync(webPackagesFile)) {
const content = fs.readFileSync(webPackagesFile, 'utf-8')
const depPackages = content.split('\n').filter((line) => line && line.startsWith('web-'))
if (depPackages.length) {
mapDependentPackagesToTestSuite(entry, depPackages)
return
}
}
// make all packages as dependent
mapDependentPackagesToTestSuite(entry, allWebPackages)
return
})

function mapDependentPackagesToTestSuite(testSuite, depPackages) {
depPackages.forEach((pkg) => {
if (!fs.existsSync(path.join(packagesDir, pkg))) {
throw new Error(
`Package "${pkg}" listed as dependent for test suite "${testSuite}" does not exist.\nPath: ${path.join(packagesDir, pkg)}`
)
}

if (!(pkg in packageToTestSuiteMap)) {
packageToTestSuiteMap[pkg] = []
}
!packageToTestSuiteMap[pkg].includes(testSuite) && packageToTestSuiteMap[pkg].push(testSuite)
})
defaultDependentPackages.forEach((pkg) => {
if (!(pkg in packageToTestSuiteMap)) {
packageToTestSuiteMap[pkg] = []
}
!packageToTestSuiteMap[pkg].includes(testSuite) && packageToTestSuiteMap[pkg].push(testSuite)
})
}

function getChangedFiles() {
const changedFiles = execSync(`git diff --name-only origin/${targetBranch} HEAD`).toString()
console.log('[INFO] Changed files:\n', changedFiles)
return [...new Set([...changedFiles.split('\n')])].filter((file) => file)
}

function getPackageFromFile(file) {
if (!file.startsWith('packages/')) {
return
}
const packages = Object.keys(packageToTestSuiteMap)
for (const pkg of packages) {
if (file.startsWith(`packages/${pkg}`)) {
return pkg
}
}
}

function getAffectedTestSuites(changedFiles) {
const affectedSuites = new Set()
for (const file of changedFiles) {
if (mandatoryPaths.some((mandatoryPath) => file.startsWith(mandatoryPath))) {
// run all test suites
return testSuites
}
const packageName = getPackageFromFile(file)
if (packageName && packageName in packageToTestSuiteMap) {
packageToTestSuiteMap[packageName].forEach((suite) => affectedSuites.add(suite))
}
}
return Array.from(affectedSuites)
}

function createSuitesToRunEnvFile(suites = []) {
console.log('[INFO] Provided test suites/features:\n - ' + suitesToCheck.join('\n - '))
console.log('[INFO] Test suites/features to run:\n - ' + suites.join('\n - '))
const envContent = ['TEST_SUITES', suites.join(',')]
if (suites[0].startsWith('cucumber/')) {
envContent[0] = ['FEATURE_FILES']
}
// create suites.env file in the same directory as the script
fs.writeFileSync(`${scriptDir}/suites.env`, envContent.join('='))
}

function main() {
const changedFiles = getChangedFiles()
if (changedFiles.length === 0) {
console.log('[INFO] No changes detected.')
process.exit(78) // Skip the pipeline
}
const affectedTestSuites = getAffectedTestSuites(changedFiles)
if (affectedTestSuites.length === 0) {
console.log('[INFO] No affected test suites/features to run.')
process.exit(78) // Skip the pipeline
}
if (suitesToCheck.length) {
const suitesToRun = suitesToCheck.filter((suite) => {
suite = suite.trim()
if (suite.startsWith('cucumber/')) {
suite = suite.replace('cucumber/features/', '').split('/').shift()
}
return affectedTestSuites.includes(suite)
})
if (suitesToRun.length === 0) {
console.log(
'[INFO] The following test suites/features are not affected and will be skipped:\n - ',
suitesToCheck.join('\n - ')
)
process.exit(78) // Skip the pipeline
}
createSuitesToRunEnvFile(suitesToRun)
return
}
createSuitesToRunEnvFile(affectedTestSuites)
}

main()
8 changes: 8 additions & 0 deletions tests/e2e/cucumber/features/admin-settings/web-packages.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
######################################################################
# USED FOR CI ONLY #
# Run the test suite only if the changes are in the listed packages. #
# #
# List of web packages that affect this test suite. #
######################################################################

web-app-admin-settings
Loading