nuclei v3 : misc updates (#4247)

* use parsed options while signing

* update project layout to v3

* fix .gitignore

* remove example template

* misc updates

* bump tlsx version

* hide template sig warning with env

* js: retain value while using log

* fix nil pointer derefernce

* misc doc update

---------

Co-authored-by: sandeep <[email protected]>

Author: tarunKoyalwar

.github/dependabot.yml

@@ -8,7 +8,7 @@ updates:
 
   # Maintain dependencies for go modules
   - package-ecosystem: "gomod"
-    directory: "v2/"
+    directory: "/"
     schedule:
       interval: "weekly"
     target-branch: "dev"

.github/workflows/build-test.yml

@@ -34,17 +34,15 @@ jobs:
         run: |
           go clean -modcache
           go mod tidy
-        working-directory: v2/
 
       - name: Build
         run: go build .
-        working-directory: v2/cmd/nuclei/
+        working-directory: cmd/nuclei/
 
       - name: Test
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         run: go test ./...
-        working-directory: v2/
 
       - name: Integration Tests
         timeout-minutes: 50
@@ -59,12 +57,12 @@ jobs:
       - name: Race Condition Tests
         if: ${{ matrix.os != 'windows-latest' }} # known issue: https://github.com/golang/go/issues/46099
         run: go run -race . -l ../functional-test/targets.txt -id tech-detect,tls-version
-        working-directory: v2/cmd/nuclei/
+        working-directory: cmd/nuclei/
 
       - name: Example SDK Simple
         run: go run .
-        working-directory: v2/examples/simple/
+        working-directory: examples/simple/
 
       - name: Example SDK Advanced
         run: go run .
-        working-directory: v2/examples/advanced/
+        working-directory: examples/advanced/

.github/workflows/functional-test.yml

@@ -30,4 +30,4 @@ jobs:
         run: |
           chmod +x run.sh
           bash run.sh ${{ matrix.os }}
-        working-directory: v2/cmd/functional-test
+        working-directory: cmd/functional-test

.github/workflows/lint-test.yml

@@ -24,5 +24,4 @@ jobs:
         uses: golangci/[email protected]
         with:
           version: latest
-          args: --timeout 5m
-          working-directory: v2/
+          args: --timeout 5m

.github/workflows/performance-test.yaml

@@ -28,9 +28,8 @@ jobs:
         run: |
           go clean -modcache
           go mod tidy
-        working-directory: v2/
 
       # Max GH exection time 6H => timeout after that
       - name: Running performance with big list
         run: go run -race . -l ../functional-test/targets-150.txt
-        working-directory: v2/cmd/nuclei/
+        working-directory: cmd/nuclei/

.github/workflows/publish-docs.yaml

@@ -31,7 +31,6 @@ jobs:
           go build -o "cmd/docgen/docgen" cmd/docgen/docgen.go 
           ./cmd/docgen/docgen ../SYNTAX-REFERENCE.md ../nuclei-jsonschema.json
           git status -s | wc -l | xargs -I {} echo CHANGES={} >> $GITHUB_OUTPUT
-        working-directory: v2
 
       - name: Commit files
         if: steps.generate-docs.outputs.CHANGES > 0

.github/workflows/release-binary.yml

@@ -23,7 +23,6 @@ jobs:
         with: 
           args: "release --rm-dist"
           version: latest
-          workdir: v2/
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}"

.github/workflows/release-test.yml

@@ -25,5 +25,4 @@ jobs:
         uses: goreleaser/goreleaser-action@v4
         with:
           args: "release --clean --snapshot"
-          version: latest
-          workdir: v2/
+          version: latest

.github/workflows/template-validate.yml

@@ -20,4 +20,4 @@ jobs:
           go run . -ut
           go run . -validate
           go run . -validate -w workflows
-        working-directory: v2/cmd/nuclei/
+        working-directory: cmd/nuclei/

.gitignore

@@ -1,27 +1,27 @@
 .idea
 .vscode
 .devcontainer
-v2/vendor
-v2/dist
+vendor
+dist
 integration_tests/nuclei
 integration_tests/integration-test
-v2/cmd/nuclei/main
-v2/cmd/nuclei/nuclei
-v2/cmd/integration-test/nuclei
-v2/cmd/functional-test/nuclei_dev
-v2/cmd/functional-test/nuclei_main
-v2/cmd/functional-test/functional-test
-v2/cmd/docgen/docgen
-v2/pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
-v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
-v2/pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
+cmd/nuclei/main
+cmd/nuclei/nuclei
+cmd/integration-test/nuclei
+cmd/functional-test/nuclei_dev
+cmd/functional-test/nuclei_main
+cmd/functional-test/functional-test
+cmd/docgen/docgen
+pkg/protocols/common/helpers/deserialization/testdata/Deserialize.class
+pkg/protocols/common/helpers/deserialization/testdata/ValueObject.class
+pkg/protocols/common/helpers/deserialization/testdata/ValueObject2.ser
 *.exe
-v2/.gitignore
-v2/pkg/js/devtools/bindgen/cmd/bindgen
-v2/pkg/js/devtools/jsdocgen/jsdocgen
-bindgen
-jsdocgen
-nuclei
-v2/scrapefuncs
+.gitignore
+pkg/js/devtools/bindgen/cmd/bindgen
+pkg/js/devtools/jsdocgen/jsdocgen
+./bindgen
+./jsdocgen
+./scrapefuncs
 *.DS_Store
-v2/pkg/protocols/headless/engine/.cache
+pkg/protocols/headless/engine/.cache
+./nuclei

v2/.goreleaser.yml .goreleaser.yml

@@ -7,9 +7,9 @@
       <env name="TESTS" value="http/dsl-functions.yaml" />
     </envs>
     <kind value="PACKAGE" />
-    <package value="github.com/projectdiscovery/nuclei/v2/cmd/integration-test" />
+    <package value="github.com/projectdiscovery/nuclei/v3/cmd/integration-test" />
     <directory value="$PROJECT_DIR$" />
-    <filePath value="$PROJECT_DIR$/v2/cmd/integration-test/integration-test.go" />
+    <filePath value="$PROJECT_DIR$/cmd/integration-test/integration-test.go" />
     <method v="2" />
   </configuration>
 </component>

.run/DSLFunctionsIT.run.xml

@@ -2,10 +2,10 @@
   <configuration default="false" name="RegressionTests" type="ShConfigurationType">
     <option name="SCRIPT_TEXT" value="" />
     <option name="INDEPENDENT_SCRIPT_PATH" value="true" />
-    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/v2/cmd/functional-test/run.sh" />
+    <option name="SCRIPT_PATH" value="$PROJECT_DIR$/cmd/functional-test/run.sh" />
     <option name="SCRIPT_OPTIONS" value="" />
     <option name="INDEPENDENT_SCRIPT_WORKING_DIRECTORY" value="true" />
-    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/v2/cmd/functional-test/" />
+    <option name="SCRIPT_WORKING_DIRECTORY" value="$PROJECT_DIR$/cmd/functional-test/" />
     <option name="INDEPENDENT_INTERPRETER_PATH" value="true" />
     <option name="INTERPRETER_PATH" value="/bin/zsh" />
     <option name="INTERPRETER_OPTIONS" value="" />

.run/RegressionTests.run.xml

@@ -4,7 +4,7 @@
     <working_directory value="$PROJECT_DIR$/v2" />
     <go_parameters value="-i" />
     <kind value="DIRECTORY" />
-    <directory value="$PROJECT_DIR$/v2/" />
+    <directory value="$PROJECT_DIR$/" />
     <filePath value="$PROJECT_DIR$" />
     <framework value="gotest" />
     <method v="2" />

.run/UnitTests.run.xml

@@ -82,9 +82,9 @@ A brief overview of the methods is provided below -
 
 For reference protocol requests implementations, one can look at the below packages  - 
 
-1. [pkg/protocols/http](./v2/pkg/protocols/http)
-2. [pkg/protocols/dns](./v2/pkg/protocols/dns)
-3. [pkg/protocols/network](./v2/pkg/protocols/network)
+1. [pkg/protocols/http](./pkg/protocols/http)
+2. [pkg/protocols/dns](./pkg/protocols/dns)
+3. [pkg/protocols/network](./pkg/protocols/network)
 
 ### Executer
 
@@ -189,9 +189,9 @@ Polling for interactions and server registration only starts when a template use
 
 Next we arrive in the `RunEnumeration` function of the runner.
 
-`HostErrorsCache` is initialised which is used throughout the run of Nuclei enumeration to keep track of errors per host and skip further requests if the errors are greater than the provided threshold. The functionality for the error tracking cache is defined in [hosterrorscache.go](https://github.com/projectdiscovery/nuclei/blob/main/v2/pkg/protocols/common/hosterrorscache/hosterrorscache.go) and is pretty simplistic in nature.
+`HostErrorsCache` is initialised which is used throughout the run of Nuclei enumeration to keep track of errors per host and skip further requests if the errors are greater than the provided threshold. The functionality for the error tracking cache is defined in [hosterrorscache.go](https://github.com/projectdiscovery/nuclei/blob/main/pkg/protocols/common/hosterrorscache/hosterrorscache.go) and is pretty simplistic in nature.
 
-Next the `WorkflowLoader` is initialised which used to load workflows. It exists in `v2/pkg/parsers/workflow_loader.go`
+Next the `WorkflowLoader` is initialised which used to load workflows. It exists in `pkg/parsers/workflow_loader.go`
 
 The loader is initialised moving forward which is responsible for Using Catalog, Passed Tags, Filters, Paths, etc. to return compiled `Templates` and `Workflows`. 
 
@@ -387,15 +387,15 @@ Almost all of these protocols have boilerplate functions for which default imple
 
 Step by step description of how to add a new protocol to Nuclei - 
 
-1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with fewer options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `v2/pkg/templates/types/types.go`.
+1. Add the protocol implementation in `pkg/protocols` directory. If it's a small protocol with fewer options, considering adding it to the `pkg/protocols/others` directory. Add the enum for the new protocol to `pkg/templates/types/types.go`.
 
 2. Add the protocol request structure to the `Template` structure fields. This is done in `pkg/templates/templates.go` with the corresponding import line.
 
 ```go
 
 import (
 	...
-	"github.com/projectdiscovery/nuclei/v2/pkg/protocols/others/websocket"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/others/websocket"
 )
 
 // Template is a YAML input file which defines all the requests and
@@ -479,52 +479,52 @@ $ go tool pprof -http=:8081 mem.pprof
 
 ## Project Structure
 
-- [v2/pkg/reporting](./v2/pkg/reporting) - Reporting modules for nuclei.
-- [v2/pkg/reporting/exporters/sarif](./v2/pkg/reporting/exporters/sarif) - Sarif Result Exporter
-- [v2/pkg/reporting/exporters/markdown](./v2/pkg/reporting/exporters/markdown) - Markdown Result Exporter
-- [v2/pkg/reporting/exporters/es](./v2/pkg/reporting/exporters/es) - Elasticsearch Result Exporter
-- [v2/pkg/reporting/dedupe](./v2/pkg/reporting/dedupe) - Dedupe module for Results
-- [v2/pkg/reporting/trackers/gitlab](./v2/pkg/reporting/trackers/gitlab) - GitLab Issue Tracker Exporter
-- [v2/pkg/reporting/trackers/jira](./v2/pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter
-- [v2/pkg/reporting/trackers/github](./v2/pkg/reporting/trackers/github) - GitHub Issue Tracker Exporter
-- [v2/pkg/reporting/format](./v2/pkg/reporting/format) - Result Formatting Functions
-- [v2/pkg/parsers](./v2/pkg/parsers) - Implements template as well as workflow loader for initial template discovery, validation and - loading.
-- [v2/pkg/types](./v2/pkg/types) - Contains CLI options as well as misc helper functions.
-- [v2/pkg/progress](./v2/pkg/progress) - Progress tracking
-- [v2/pkg/operators](./v2/pkg/operators) - Operators for Nuclei
-- [v2/pkg/operators/common/dsl](./v2/pkg/operators/common/dsl) - DSL functions for Nuclei YAML Syntax
-- [v2/pkg/operators/matchers](./v2/pkg/operators/matchers) - Matchers implementation
-- [v2/pkg/operators/extractors](./v2/pkg/operators/extractors) - Extractors implementation
-- [v2/pkg/catalog](./v2/pkg/catalog) - Template loading from disk helpers
-- [v2/pkg/catalog/config](./v2/pkg/catalog/config) - Internal configuration management
-- [v2/pkg/catalog/loader](./v2/pkg/catalog/loader) - Implements loading and validation of templates and workflows.
-- [v2/pkg/catalog/loader/filter](./v2/pkg/catalog/loader/filter) - Filter filters templates based on tags and paths
-- [v2/pkg/output](./v2/pkg/output) - Output module for nuclei
-- [v2/pkg/workflows](./v2/pkg/workflows) - Workflow execution logic + declarations
-- [v2/pkg/utils](./v2/pkg/utils) - Utility functions
-- [v2/pkg/model](./v2/pkg/model) - Template Info + misc
-- [v2/pkg/templates](./v2/pkg/templates) - Templates core starting point
-- [v2/pkg/templates/cache](./v2/pkg/templates/cache) - Templates cache
-- [v2/pkg/protocols](./v2/pkg/protocols) - Protocol Specification
-- [v2/pkg/protocols/file](./v2/pkg/protocols/file) - File protocol
-- [v2/pkg/protocols/network](./v2/pkg/protocols/network) - Network protocol
-- [v2/pkg/protocols/common/expressions](./v2/pkg/protocols/common/expressions) - Expression evaluation + Templating Support
-- [v2/pkg/protocols/common/interactsh](./v2/pkg/protocols/common/interactsh) - Interactsh integration
-- [v2/pkg/protocols/common/generators](./v2/pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc.)
-- [v2/pkg/protocols/common/executer](./v2/pkg/protocols/common/executer) - Default Template Executer
-- [v2/pkg/protocols/common/replacer](./v2/pkg/protocols/common/replacer) - Template replacement helpers
-- [v2/pkg/protocols/common/helpers/eventcreator](./v2/pkg/protocols/common/helpers/eventcreator) - Result event creator
-- [v2/pkg/protocols/common/helpers/responsehighlighter](./v2/pkg/protocols/common/helpers/responsehighlighter) - Debug response highlighter
-- [v2/pkg/protocols/common/helpers/deserialization](./v2/pkg/protocols/common/helpers/deserialization) - Deserialization helper functions
-- [v2/pkg/protocols/common/hosterrorscache](./v2/pkg/protocols/common/hosterrorscache) - Host errors cache for tracking erroring hosts
-- [v2/pkg/protocols/offlinehttp](./v2/pkg/protocols/offlinehttp) - Offline http protocol
-- [v2/pkg/protocols/http](./v2/pkg/protocols/http) - HTTP protocol
-- [v2/pkg/protocols/http/race](./v2/pkg/protocols/http/race) - HTTP Race Module
-- [v2/pkg/protocols/http/raw](./v2/pkg/protocols/http/raw) - HTTP Raw Request Support
-- [v2/pkg/protocols/headless](./v2/pkg/protocols/headless) - Headless Module
-- [v2/pkg/protocols/headless/engine](./v2/pkg/protocols/headless/engine) - Internal Headless implementation
-- [v2/pkg/protocols/dns](./v2/pkg/protocols/dns) - DNS protocol
-- [v2/pkg/projectfile](./v2/pkg/projectfile) - Project File Implementation
+- [pkg/reporting](./pkg/reporting) - Reporting modules for nuclei.
+- [pkg/reporting/exporters/sarif](./pkg/reporting/exporters/sarif) - Sarif Result Exporter
+- [pkg/reporting/exporters/markdown](./pkg/reporting/exporters/markdown) - Markdown Result Exporter
+- [pkg/reporting/exporters/es](./pkg/reporting/exporters/es) - Elasticsearch Result Exporter
+- [pkg/reporting/dedupe](./pkg/reporting/dedupe) - Dedupe module for Results
+- [pkg/reporting/trackers/gitlab](./pkg/reporting/trackers/gitlab) - GitLab Issue Tracker Exporter
+- [pkg/reporting/trackers/jira](./pkg/reporting/trackers/jira) - Jira Issue Tracker Exporter
+- [pkg/reporting/trackers/github](./pkg/reporting/trackers/github) - GitHub Issue Tracker Exporter
+- [pkg/reporting/format](./pkg/reporting/format) - Result Formatting Functions
+- [pkg/parsers](./pkg/parsers) - Implements template as well as workflow loader for initial template discovery, validation and - loading.
+- [pkg/types](./pkg/types) - Contains CLI options as well as misc helper functions.
+- [pkg/progress](./pkg/progress) - Progress tracking
+- [pkg/operators](./pkg/operators) - Operators for Nuclei
+- [pkg/operators/common/dsl](./pkg/operators/common/dsl) - DSL functions for Nuclei YAML Syntax
+- [pkg/operators/matchers](./pkg/operators/matchers) - Matchers implementation
+- [pkg/operators/extractors](./pkg/operators/extractors) - Extractors implementation
+- [pkg/catalog](./pkg/catalog) - Template loading from disk helpers
+- [pkg/catalog/config](./pkg/catalog/config) - Internal configuration management
+- [pkg/catalog/loader](./pkg/catalog/loader) - Implements loading and validation of templates and workflows.
+- [pkg/catalog/loader/filter](./pkg/catalog/loader/filter) - Filter filters templates based on tags and paths
+- [pkg/output](./pkg/output) - Output module for nuclei
+- [pkg/workflows](./pkg/workflows) - Workflow execution logic + declarations
+- [pkg/utils](./pkg/utils) - Utility functions
+- [pkg/model](./pkg/model) - Template Info + misc
+- [pkg/templates](./pkg/templates) - Templates core starting point
+- [pkg/templates/cache](./pkg/templates/cache) - Templates cache
+- [pkg/protocols](./pkg/protocols) - Protocol Specification
+- [pkg/protocols/file](./pkg/protocols/file) - File protocol
+- [pkg/protocols/network](./pkg/protocols/network) - Network protocol
+- [pkg/protocols/common/expressions](./pkg/protocols/common/expressions) - Expression evaluation + Templating Support
+- [pkg/protocols/common/interactsh](./pkg/protocols/common/interactsh) - Interactsh integration
+- [pkg/protocols/common/generators](./pkg/protocols/common/generators) - Payload support for Requests (Sniper, etc.)
+- [pkg/protocols/common/executer](./pkg/protocols/common/executer) - Default Template Executer
+- [pkg/protocols/common/replacer](./pkg/protocols/common/replacer) - Template replacement helpers
+- [pkg/protocols/common/helpers/eventcreator](./pkg/protocols/common/helpers/eventcreator) - Result event creator
+- [pkg/protocols/common/helpers/responsehighlighter](./pkg/protocols/common/helpers/responsehighlighter) - Debug response highlighter
+- [pkg/protocols/common/helpers/deserialization](./pkg/protocols/common/helpers/deserialization) - Deserialization helper functions
+- [pkg/protocols/common/hosterrorscache](./pkg/protocols/common/hosterrorscache) - Host errors cache for tracking erroring hosts
+- [pkg/protocols/offlinehttp](./pkg/protocols/offlinehttp) - Offline http protocol
+- [pkg/protocols/http](./pkg/protocols/http) - HTTP protocol
+- [pkg/protocols/http/race](./pkg/protocols/http/race) - HTTP Race Module
+- [pkg/protocols/http/raw](./pkg/protocols/http/raw) - HTTP Raw Request Support
+- [pkg/protocols/headless](./pkg/protocols/headless) - Headless Module
+- [pkg/protocols/headless/engine](./pkg/protocols/headless/engine) - Internal Headless implementation
+- [pkg/protocols/dns](./pkg/protocols/dns) - DNS protocol
+- [pkg/projectfile](./pkg/projectfile) - Project File Implementation
 
 ### Notes
 

DESIGN.md

@@ -3,14 +3,14 @@ FROM golang:1.21-alpine AS build-env
 RUN apk add build-base
 WORKDIR /app
 COPY . /app
-WORKDIR /app/v2
+WORKDIR /app
 RUN go mod download
 RUN go build ./cmd/nuclei
 
 # Release
 FROM alpine:3.18.2
 RUN apk -U upgrade --no-cache \
     && apk add --no-cache bind-tools chromium ca-certificates
-COPY --from=build-env /app/v2/nuclei /usr/local/bin/
+COPY --from=build-env /app/nuclei /usr/local/bin/
 
 ENTRYPOINT ["nuclei"]

Dockerfile

@@ -25,7 +25,7 @@ docs:
 test:
 	$(GOTEST) $(GOFLAGS) ./...
 integration:
-	cd ../integration_tests; bash run.sh
+	cd integration_tests; bash run.sh
 functional:
 	cd cmd/functional-test; bash run.sh
 tidy:

v2/Makefile Makefile

@@ -59,7 +59,7 @@ We have a [dedicated repository](https://github.com/projectdiscovery/nuclei-temp
 Nuclei requires **go1.21** to install successfully. Run the following command to install the latest version -
 
 ```sh
-go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
+go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
 ```
 
 <details>
@@ -379,7 +379,7 @@ We have [a discussion thread around this](https://github.com/projectdiscovery/nu
 
 ### Using Nuclei From Go Code
 
-Complete guide of using Nuclei as Library/SDK is available at [lib](v2/lib/README.md)
+Complete guide of using Nuclei as Library/SDK is available at [godoc](https://pkg.go.dev/github.com/projectdiscovery/nuclei/v3/lib#section-readme)
 
 
 ### Resources

README.md

@@ -55,7 +55,7 @@ Nuclei使用零误报的定制模板向目标发送请求，同时可以对主
 Nuclei需要**go1**才能安装成功。执行下列命令安装最新版本的Nuclei
 
 ```sh
-go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
+go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
 ```
 
 **更多的安装方式 [请点击此处](https://nuclei.projectdiscovery.io/nuclei/get-started/).**

README_CN.md

@@ -55,7 +55,7 @@ Kami memiliki [repositori khusus](https://github.com/projectdiscovery/nuclei-tem
 Nuclei membutuhkan **go1.21** agar dapat diinstall. Jalankan perintah berikut untuk menginstal versi terbaru -
 
 ```sh
-go install -v github.com/projectdiscovery/nuclei/v2/cmd/nuclei@latest
+go install -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@latest
 ```
 
 **Metode [instalasi lain dapat ditemukan di sini](https://nuclei.projectdiscovery.io/nuclei/get-started/).**