Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions packages/db/src/services/chart.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -600,11 +600,14 @@ export async function getChartSql({
return buildInlineCohortJoin(cohortId, projectId, 'e', cohortMeta);
}).join(' ');

// Determine data source: use custom event CTE if present, otherwise events table
const dataSource = customEvent ? 'custom_event_data' : TABLE_NAMES.events;

// Add top_breakdowns CTE using the builder
addCte(
'top_breakdowns',
`SELECT ${breakdownSelects}
FROM ${TABLE_NAMES.events} e
FROM ${dataSource} AS e
${profilesJoinRef ? `${profilesJoinRef} ` : ''}${cohortJoinsForTop ? `${cohortJoinsForTop} ` : ''}${getWhereWithoutBar()}
GROUP BY ${breakdownSelects}
ORDER BY count(*) DESC
Expand Down Expand Up @@ -685,19 +688,22 @@ export async function getChartSql({

const totalCountWhere = getWhereWithoutBar();

// Determine data source: use custom event CTE if present, otherwise events table
const dataSourceForBreakdown = customEvent ? 'custom_event_data' : TABLE_NAMES.events;

// Build cohort JOINs for breakdown_totals CTE
// NOTE: ClickHouse CTEs cannot reference other CTEs in JOINs, so we inline the subquery
const cohortJoinsForBreakdown = cohortIds.map((cohortId) => {
const cohortMeta = cohortMetadata.get(cohortId);
return buildInlineCohortJoin(cohortId, projectId, TABLE_NAMES.events, cohortMeta);
return buildInlineCohortJoin(cohortId, projectId, dataSourceForBreakdown, cohortMeta);
}).join(' ');

addCte(
'breakdown_totals',
`SELECT
${breakdownSelects},
uniq(${TABLE_NAMES.events}.profile_id) as total_count
FROM ${TABLE_NAMES.events}
uniq(${dataSourceForBreakdown}.profile_id) as total_count
FROM ${dataSourceForBreakdown}
${profilesJoinRefForCTE ? `${profilesJoinRefForCTE} ` : ''}${cohortJoinsForBreakdown ? `${cohortJoinsForBreakdown} ` : ''}${totalCountWhere}
GROUP BY ${breakdownGroupBy}`,
);
Expand All @@ -714,17 +720,20 @@ export async function getChartSql({
} else {
const totalCountWhere = getWhereWithoutBar();

// Determine data source: use custom event CTE if present, otherwise events table
const dataSourceForTotal = customEvent ? 'custom_event_data' : TABLE_NAMES.events;

// Build cohort JOINs for total_unique CTE
// NOTE: ClickHouse CTEs cannot reference other CTEs in JOINs, so we inline the subquery
const cohortJoinsForTotal = cohortIds.map((cohortId) => {
const cohortMeta = cohortMetadata.get(cohortId);
return buildInlineCohortJoin(cohortId, projectId, TABLE_NAMES.events, cohortMeta);
return buildInlineCohortJoin(cohortId, projectId, dataSourceForTotal, cohortMeta);
}).join(' ');

addCte(
'total_unique',
`SELECT uniq(${TABLE_NAMES.events}.profile_id) as total_count
FROM ${TABLE_NAMES.events}
`SELECT uniq(${dataSourceForTotal}.profile_id) as total_count
FROM ${dataSourceForTotal}
${profilesJoinRefForCTE ? `${profilesJoinRefForCTE} ` : ''}${cohortJoinsForTotal ? `${cohortJoinsForTotal} ` : ''}${totalCountWhere}`,
);

Expand Down
2 changes: 1 addition & 1 deletion packages/db/src/services/conversion.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ export class ConversionService {
// Build funnel conditions for all events
const conditions = events.map(event => {
const where = Object.values(
getEventFiltersWhereClause(event.filters),
getEventFiltersWhereClause(event.filters, projectId),
).join(' AND ');

return where
Expand Down
41 changes: 38 additions & 3 deletions packages/db/src/services/custom-event.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,17 @@ function buildCustomEventSourceQuery(
}

/**
* Expand a custom event into SQL UNION of source events
* This creates a CTE that can be used in place of the events table
* Expand a custom event into SQL query
*
* PERFORMANCE OPTIMIZATION:
* When all source events have no filters, we use a single SELECT with IN clause
* instead of UNION ALL. This reduces N table scans to 1 table scan.
*
* Example:
* - Before: 20 separate SELECTs with UNION ALL (20 table scans)
* - After: 1 SELECT with IN ('event1', 'event2', ..., 'event20') (1 table scan)
*
* For events with filters, we fall back to UNION ALL to ensure correct filtering.
*
* @param customEvent - The custom event definition
* @param baseWhere - Base WHERE conditions to apply to all source events (date ranges, etc)
Expand All @@ -81,6 +90,33 @@ export function expandCustomEventToSQL(
): string {
const definition = customEvent.definition;

// Check if we can use the optimized path
// Optimization is only safe when all events have no filters
const canOptimize = definition.events.every(
(event) => !event.filters || event.filters.length === 0
);

if (canOptimize && definition.events.length > 0) {
// OPTIMIZED PATH: Single SELECT with IN clause
// This reduces N table scans to 1 table scan, dramatically improving performance
const eventNames = definition.events.map((e) => sqlstring.escape(e.name));

const whereClauses = [
`project_id = ${sqlstring.escape(customEvent.projectId)}`,
`name IN (${eventNames.join(', ')})`,
...baseWhere,
];

return `
SELECT * REPLACE(${sqlstring.escape(customEvent.name)} AS name)
FROM ${TABLE_NAMES.events}
WHERE ${whereClauses.join(' AND ')}
`;
}

// FALLBACK PATH: UNION ALL for events with filters
// When events have filters, each event may need different WHERE conditions,
// so we must use UNION ALL to apply filters correctly per event
const sourceQueries = definition.events.map((sourceEvent) =>
buildCustomEventSourceQuery(
customEvent.projectId,
Expand All @@ -90,6 +126,5 @@ export function expandCustomEventToSQL(
),
);

// UNION ALL for OR logic (match any source event)
return sourceQueries.join(' UNION ALL ');
}
6 changes: 3 additions & 3 deletions packages/db/src/services/funnel.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,10 @@ export class FunnelService {
};
}

getFunnelConditions(events: IChartEvent[] = []): string[] {
getFunnelConditions(events: IChartEvent[] = [], projectId?: string): string[] {
return events.map((event) => {
const { sb, getWhere } = createSqlBuilder();
sb.where = getEventFiltersWhereClause(event.filters);
sb.where = getEventFiltersWhereClause(event.filters, projectId);
sb.where.name = `name = ${sqlstring.escape(event.name)}`;
return getWhere().replace('WHERE ', '');
});
Expand Down Expand Up @@ -147,7 +147,7 @@ export class FunnelService {
fromClause: string;
needsNameFilter: boolean;
}) {
const funnels = this.getFunnelConditions(eventSeries);
const funnels = this.getFunnelConditions(eventSeries, projectId);

const query = clix(this.client, timezone)
.select([
Expand Down