Implement shared parser with compiled cache functionality and related tests

lib/sdk_private.go

@@ -3,6 +3,7 @@
 import (
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -37,6 +38,21 @@
 	"github.com/projectdiscovery/ratelimit"
 )
 
+// enginePool provides shared parsed-cache across engines (opt-in)
+var (
+	sharedParsedOnce sync.Once
+	sharedParsed     *templates.Cache
+)
+
+func getSharedParser() *templates.Parser {
+	// Initialize the shared parsed cache once
+	sharedParsedOnce.Do(func() {
+		sharedParsed = templates.NewCache()
+	})
+	// Return a fresh Parser each call that reuses only the shared parsed cache,
+	return templates.NewParserWithParsedCache(sharedParsed)
+}
+
 // applyRequiredDefaults to options
 func (e *NucleiEngine) applyRequiredDefaults(ctx context.Context) {
 	mockoutput := testutils.NewMockOutputWriter(e.opts.OmitTemplate)
@@ -123,7 +139,12 @@
 	}
 
 	if e.parser == nil {
-		e.parser = templates.NewParser()
+		//TODO: remove this feature flag after testing
+		if os.Getenv("NUCLEI_USE_SHARED_COMPILED") == "1" {
+			e.parser = templates.NewSharedParserWithCompiledCache()
+		} else {
+			e.parser = templates.NewParser()
+		}
 	}
 
 	if protocolstate.ShouldInit(e.opts.ExecutionId) {

lib/sdk_test.go

@@ -3,6 +3,7 @@
 import (
 	"context"
 	"log"
+	"os"
 	"testing"
 	"time"
 
@@ -35,3 +36,28 @@
 	}
 	defer ne.Close()
 }
+
+func TestSharedParserOptIn(t *testing.T) {
+	os.Setenv("NUCLEI_USE_SHARED_PARSER", "1")
+	t.Cleanup(func() { os.Unsetenv("NUCLEI_USE_SHARED_PARSER") })
+
+	ne, err := nuclei.NewNucleiEngineCtx(context.Background())
+	if err != nil {
+		t.Fatalf("engine error: %v", err)
+	}
+	p := ne.GetParser()
+	if p == nil {
+		t.Fatalf("expected templates.Parser")
+	}
+	ne2, err := nuclei.NewNucleiEngineCtx(context.Background())
+	if err != nil {
+		t.Fatalf("engine2 error: %v", err)
+	}
+	p2 := ne2.GetParser()
+	if p2 == nil {
+		t.Fatalf("expected templates.Parser2")
+	}
+	if p.Cache() != p2.Cache() {
+		t.Fatalf("expected shared parsed cache across engines when opt-in is set")
+	}
+}

pkg/templates/compile.go

@@ -224,11 +224,31 @@ func Parse(filePath string, preprocessor Preprocessor, options *protocols.Execut
 	}
 	template.Path = filePath
 	if !options.DoNotCache {
-		parser.compiledTemplatesCache.Store(filePath, template, nil, err)
+		// Store a sanitized template in compiled cache to avoid retaining engine-scoped state.
+		cacheTpl := *template
+		cacheTpl.Options = sanitizeOptionsForCache(template.Options)
+		// Raw template bytes are not needed in compiled cache; keep per-engine
+		cacheTpl.Options.RawTemplate = nil
+		parser.compiledTemplatesCache.Store(filePath, &cacheTpl, nil, err)
 	}
 	return template, nil
 }
 
+// sanitizeOptionsForCache strips engine-scoped fields from ExecutorOptions to avoid
+// retaining per-engine references in the shared compiled cache.
+func sanitizeOptionsForCache(src *protocols.ExecutorOptions) *protocols.ExecutorOptions {
+	if src == nil {
+		return nil
+	}
+	return &protocols.ExecutorOptions{
+		// Intentionally exclude TemplateID/Path/Verifier and RawTemplate to avoid engine leakage
+		StopAtFirstMatch: src.StopAtFirstMatch,
+		ProtocolType:     src.ProtocolType,
+		Flow:             src.Flow,
+		IsMultiProtocol:  src.IsMultiProtocol,
+	}
+}
+
 // isGlobalMatchersEnabled checks if any of requests in the template
 // have global matchers enabled. It iterates through all requests and
 // returns true if at least one request has global matchers enabled;

pkg/templates/compile_test.go

@@ -206,3 +206,161 @@ func TestWrongWorkflow(t *testing.T) {
 	require.Nil(t, got, "could not parse template")
 	require.ErrorContains(t, err, "workflows cannot have other protocols")
 }
+
+func Test_SharedCompiledCache_SharedAcrossParsers(t *testing.T) {
+	setup()
+	p1 := templates.NewSharedParserWithCompiledCache()
+	p2 := templates.NewSharedParserWithCompiledCache()
+
+	exec1 := &protocols.ExecutorOptions{
+		Output:      testutils.NewMockOutputWriter(testutils.DefaultOptions.OmitTemplate),
+		Options:     testutils.DefaultOptions,
+		Progress:    executerOpts.Progress,
+		Catalog:     executerOpts.Catalog,
+		RateLimiter: executerOpts.RateLimiter,
+		Parser:      p1,
+	}
+	// reinit options fully for isolation
+	opts2 := testutils.DefaultOptions
+	testutils.Init(opts2)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
+	exec2 := &protocols.ExecutorOptions{
+		Output:      testutils.NewMockOutputWriter(opts2.OmitTemplate),
+		Options:     opts2,
+		Progress:    progressImpl,
+		Catalog:     executerOpts.Catalog,
+		RateLimiter: executerOpts.RateLimiter,
+		Parser:      p2,
+	}
+
+	filePath := "tests/match-1.yaml"
+
+	got1, err := templates.Parse(filePath, nil, exec1)
+	require.NoError(t, err)
+	require.NotNil(t, got1)
+
+	got2, err := templates.Parse(filePath, nil, exec2)
+	require.NoError(t, err)
+	require.NotNil(t, got2)
+
+	require.Equal(t, p1.CompiledCache(), p2.CompiledCache())
+	require.Greater(t, p1.CompiledCount(), 0)
+	require.Equal(t, p1.CompiledCount(), p2.CompiledCount())
+}
+
+func Test_SharedCompiledCache_OptionsIsolation(t *testing.T) {
+	setup()
+	p1 := templates.NewSharedParserWithCompiledCache()
+	p2 := templates.NewSharedParserWithCompiledCache()
+
+	exec1 := &protocols.ExecutorOptions{
+		Output:      testutils.NewMockOutputWriter(testutils.DefaultOptions.OmitTemplate),
+		Options:     testutils.DefaultOptions,
+		Progress:    executerOpts.Progress,
+		Catalog:     executerOpts.Catalog,
+		RateLimiter: executerOpts.RateLimiter,
+		Parser:      p1,
+	}
+	// reinit options fully for isolation
+	opts2 := testutils.DefaultOptions
+	testutils.Init(opts2)
+	progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0)
+	exec2 := &protocols.ExecutorOptions{
+		Output:      testutils.NewMockOutputWriter(opts2.OmitTemplate),
+		Options:     opts2,
+		Progress:    progressImpl,
+		Catalog:     executerOpts.Catalog,
+		RateLimiter: executerOpts.RateLimiter,
+		Parser:      p2,
+	}
+
+	filePath := "tests/match-1.yaml"
+
+	got1, err := templates.Parse(filePath, nil, exec1)
+	require.NoError(t, err)
+	require.NotNil(t, got1)
+
+	got2, err := templates.Parse(filePath, nil, exec2)
+	require.NoError(t, err)
+	require.NotNil(t, got2)
+
+	require.NotEqual(t, got1.Options, got2.Options)
+}
+
+// compiled cache does not retain engine-scoped fields
+func Test_CompiledCache_SanitizesOptions(t *testing.T) {
+	setup()
+	p := templates.NewSharedParserWithCompiledCache()
+	exec := executerOpts
+	exec.Parser = p
+	filePath := "tests/match-1.yaml"
+
+	got, err := templates.Parse(filePath, nil, exec)
+	require.NoError(t, err)
+	require.NotNil(t, got)
+
+	cached, raw, err := p.CompiledCache().Has(filePath)
+	require.NoError(t, err)
+	require.NotNil(t, cached)
+	require.Nil(t, raw)
+
+	// cached template must not hold engine-scoped references
+	require.Nil(t, cached.Options.Options)
+	require.Empty(t, cached.Options.TemplateVerifier)
+	require.Empty(t, cached.Options.TemplateID)
+	require.Empty(t, cached.Options.TemplatePath)
+	require.False(t, cached.Options.StopAtFirstMatch)
+}
+
+// different engines see different Options pointers
+func Test_EngineIsolation_NoCrossLeaks(t *testing.T) {
+	setup()
+	p1 := templates.NewSharedParserWithCompiledCache()
+	p2 := templates.NewSharedParserWithCompiledCache()
+
+	// engine 1
+	exec1 := &protocols.ExecutorOptions{
+		Output:      executerOpts.Output,
+		Options:     executerOpts.Options,
+		Progress:    executerOpts.Progress,
+		Catalog:     executerOpts.Catalog,
+		RateLimiter: executerOpts.RateLimiter,
+		Parser:      p1,
+	}
+	// engine 2 with a fresh options instance
+	opts2 := testutils.DefaultOptions
+	testutils.Init(opts2)
+	progress2, _ := progress.NewStatsTicker(0, false, false, false, 0)
+	exec2 := &protocols.ExecutorOptions{
+		Output:      testutils.NewMockOutputWriter(opts2.OmitTemplate),
+		Options:     opts2,
+		Progress:    progress2,
+		Catalog:     executerOpts.Catalog,
+		RateLimiter: executerOpts.RateLimiter,
+		Parser:      p2,
+	}
+
+	filePath := "tests/match-1.yaml"
+
+	got1, err := templates.Parse(filePath, nil, exec1)
+	require.NoError(t, err)
+	got2, err := templates.Parse(filePath, nil, exec2)
+	require.NoError(t, err)
+
+	// template options must be distinct per engine
+	require.NotEqual(t, got1.Options, got2.Options)
+
+	// http request options must bind to engine-specific ExecutorOptions copies (not shared)
+	require.NotEmpty(t, got1.RequestsHTTP)
+	require.NotEmpty(t, got2.RequestsHTTP)
+	r1 := got1.RequestsHTTP[0]
+	r2 := got2.RequestsHTTP[0]
+	// ensure options structs are not the same pointer
+	require.NotSame(t, r1.Options().Options, r2.Options().Options)
+	// mutate engine2 options and ensure it doesn't affect engine1
+	r2.Options().Options.RateLimit = 999
+	require.NotEqual(t, r1.Options().Options.RateLimit, r2.Options().Options.RateLimit)
+
+	// compiled cache instance shared, but without engine leakage
+	require.Equal(t, p1.CompiledCache(), p2.CompiledCache())
+}

pkg/templates/parser.go

@@ -29,20 +29,42 @@ type Parser struct {
 	sync.Mutex
 }
 
-func NewParser() *Parser {
-	p := &Parser{
-		parsedTemplatesCache:   NewCache(),
-		compiledTemplatesCache: NewCache(),
-	}
+var (
+	sharedParsedCacheOnce sync.Once
+	sharedParsedCache     *Cache
+)
 
-	return p
+var (
+	sharedCompiledCacheOnce sync.Once
+	sharedCompiledCache     *Cache
+)
+
+// NewParser returns a new parser with a fresh cache
+func NewParser() *Parser {
+	return &Parser{parsedTemplatesCache: NewCache(), compiledTemplatesCache: NewCache()}
 }
 
+// NewParserWithParsedCache returns a parser using provided cache
 func NewParserWithParsedCache(cache *Cache) *Parser {
-	return &Parser{
-		parsedTemplatesCache:   cache,
-		compiledTemplatesCache: NewCache(),
-	}
+	return &Parser{parsedTemplatesCache: cache, compiledTemplatesCache: NewCache()}
+}
+
+// NewSharedParser returns a parser backed by a process-wide shared parsed cache.
+// Safe for concurrent use since Cache is concurrency-safe.
+func NewSharedParser() *Parser {
+	sharedParsedCacheOnce.Do(func() {
+		sharedParsedCache = NewCache()
+	})
+	return &Parser{parsedTemplatesCache: sharedParsedCache, compiledTemplatesCache: NewCache()}
+}
+
+// NewSharedParserWithCompiledCache returns a parser backed by process-wide shared
+// parsed and compiled caches. Intended for scenarios where compiled executers
+// can be safely reused across engines by copying option-bearing fields.
+func NewSharedParserWithCompiledCache() *Parser {
+	sharedParsedCacheOnce.Do(func() { sharedParsedCache = NewCache() })
+	sharedCompiledCacheOnce.Do(func() { sharedCompiledCache = NewCache() })
+	return &Parser{parsedTemplatesCache: sharedParsedCache, compiledTemplatesCache: sharedCompiledCache}
 }
 
 // Cache returns the parsed templates cache

pkg/templates/parser_test.go

@@ -3,6 +3,7 @@ package templates
 import (
 	"errors"
 	"fmt"
+	"sync"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/disk"
@@ -156,3 +157,31 @@ func TestLoadTemplate(t *testing.T) {
 		}
 	})
 }
+
+func TestNewSharedParserSharesCache(t *testing.T) {
+	p1 := NewSharedParser()
+	p2 := NewSharedParser()
+	if p1.Cache() != p2.Cache() {
+		t.Fatalf("expected shared cache instance")
+	}
+}
+
+func TestNewSharedParserConcurrency(t *testing.T) {
+	var wg sync.WaitGroup
+	const goroutines = 50
+	parsers := make([]*Parser, goroutines)
+	wg.Add(goroutines)
+	for i := 0; i < goroutines; i++ {
+		go func(i int) {
+			defer wg.Done()
+			parsers[i] = NewSharedParser()
+		}(i)
+	}
+	wg.Wait()
+	base := parsers[0].Cache()
+	for i := 1; i < goroutines; i++ {
+		if parsers[i].Cache() != base {
+			t.Fatalf("expected all parsers to share the same cache")
+		}
+	}
+}