Feature/performance

.github/workflows/docker-build.yml

@@ -146,6 +146,7 @@ jobs:
             DATABASE_URL=postgresql://dummy:dummy@localhost:5432/dummy
 
       - name: Create/Update API tag
+        if: github.ref == 'refs/heads/main'
         run: |
           # Delete existing tag if it exists
           if git tag -l "api" | grep -q "api"; then
@@ -206,6 +207,7 @@ jobs:
             DATABASE_URL=postgresql://dummy:dummy@localhost:5432/dummy
 
       - name: Create/Update Worker tag
+        if: github.ref == 'refs/heads/main'
         run: |
           # Delete existing tag if it exists
           if git tag -l "worker" | grep -q "worker"; then
@@ -266,6 +268,7 @@ jobs:
             NO_CLOUDFLARE=1
 
       - name: Create/Update Dashboard tag
+        if: github.ref == 'refs/heads/main'
         run: |
           # Delete existing tag if it exists
           if git tag -l "dashboard" | grep -q "dashboard"; then

apps/api/src/controllers/live.controller.ts

@@ -4,7 +4,7 @@ import superjson from 'superjson';
 import type { WebSocket } from '@fastify/websocket';
 import {
   eventBuffer,
-  getProfileByIdCached,
+  getProfileById,
   transformMinimalEvent,
 } from '@openpanel/db';
 import { setSuperJson } from '@openpanel/json';
@@ -92,10 +92,7 @@ export async function wsProjectEvents(
     type,
     async (event) => {
       if (event.projectId === params.projectId) {
-        const profile = await getProfileByIdCached(
-          event.profileId,
-          event.projectId,
-        );
+        const profile = await getProfileById(event.profileId, event.projectId);
         socket.send(
           superjson.stringify(
             access

apps/api/src/controllers/profile.controller.ts

@@ -17,14 +17,14 @@ export async function updateProfile(
   }>,
   reply: FastifyReply,
 ) {
-  const { profileId, properties, ...rest } = request.body;
+  const payload = request.body;
   const projectId = request.client!.projectId;
   if (!projectId) {
     return reply.status(400).send('No projectId');
   }
   const ip = getClientIp(request)!;
   const ua = request.headers['user-agent']!;
-  const uaInfo = parseUserAgent(ua, properties);
+  const uaInfo = parseUserAgent(ua, payload.properties);
   const geo = await getGeoLocation(ip);
 
   if (
@@ -40,18 +40,28 @@ export async function updateProfile(
   }
 
   await upsertProfile({
-    id: profileId,
+    ...payload,
+    id: payload.profileId,
     isExternal: true,
     projectId,
     properties: {
-      ...(properties ?? {}),
-      ...(ip ? geo : {}),
-      ...uaInfo,
+      ...(payload.properties ?? {}),
+      country: geo.country,
+      city: geo.city,
+      region: geo.region,
+      longitude: geo.longitude,
+      latitude: geo.latitude,
+      os: uaInfo.os,
+      os_version: uaInfo.osVersion,
+      browser: uaInfo.browser,
+      browser_version: uaInfo.browserVersion,
+      device: uaInfo.device,
+      brand: uaInfo.brand,
+      model: uaInfo.model,
     },
-    ...rest,
   });
 
-  reply.status(202).send(profileId);
+  reply.status(202).send(payload.profileId);
 }
 
 export async function incrementProfileProperty(

apps/api/src/controllers/track.controller.ts

@@ -323,8 +323,18 @@ async function identify({
     projectId,
     properties: {
       ...(payload.properties ?? {}),
-      ...(geo ?? {}),
-      ...uaInfo,
+      country: geo.country,
+      city: geo.city,
+      region: geo.region,
+      longitude: geo.longitude,
+      latitude: geo.latitude,
+      os: uaInfo.os,
+      os_version: uaInfo.osVersion,
+      browser: uaInfo.browser,
+      browser_version: uaInfo.browserVersion,
+      device: uaInfo.device,
+      brand: uaInfo.brand,
+      model: uaInfo.model,
     },
   });
 }

apps/api/src/hooks/ip.hook.ts

@@ -1,9 +1,5 @@
 import { getClientIp } from '@/utils/get-client-ip';
-import type {
-  FastifyReply,
-  FastifyRequest,
-  HookHandlerDoneFunction,
-} from 'fastify';
+import type { FastifyRequest } from 'fastify';
 
 export async function ipHook(request: FastifyRequest) {
   const ip = getClientIp(request);

apps/api/src/utils/auth.ts

@@ -3,6 +3,7 @@ import type { FastifyRequest, RawRequestDefaultExpression } from 'fastify';
 import { verifyPassword } from '@openpanel/common/server';
 import type { IServiceClientWithProject } from '@openpanel/db';
 import { ClientType, getClientByIdCached } from '@openpanel/db';
+import { getCache } from '@openpanel/redis';
 import type { PostEventPayload, TrackHandlerPayload } from '@openpanel/sdk';
 import type {
   IProjectFilterIp,
@@ -131,7 +132,13 @@ export async function validateSdkRequest(
   }
 
   if (client.secret && clientSecret) {
-    if (await verifyPassword(clientSecret, client.secret)) {
+    const isVerified = await getCache(
+      `client:auth:${clientId}:${clientSecret.slice(0, 5)}`,
+      60 * 5,
+      async () => await verifyPassword(clientSecret, client.secret!),
+      true,
+    );
+    if (isVerified) {
       return client;
     }
   }

apps/start/package.json

@@ -20,7 +20,6 @@
   },
   "dependencies": {
     "@ai-sdk/react": "^1.2.5",
-    "@clickhouse/client": "^1.2.0",
     "@dnd-kit/core": "^6.3.1",
     "@dnd-kit/sortable": "^10.0.0",
     "@dnd-kit/utilities": "^3.2.2",

apps/start/src/components/events/table/columns.tsx

@@ -54,18 +54,6 @@ export function useColumns() {
           return name.replace(/_/g, ' ');
         };
 
-        const renderDuration = () => {
-          if (name === 'screen_view') {
-            return (
-              <span className="text-muted-foreground">
-                {number.shortWithUnit(duration / 1000, 'min')}
-              </span>
-            );
-          }
-
-          return null;
-        };
-
         return (
           <div className="flex items-center gap-2">
             <button
@@ -97,7 +85,6 @@ export function useColumns() {
               >
                 {renderName()}
               </button>
-              {renderDuration()}
             </span>
           </div>
         );

apps/start/src/components/ui/data-table/data-table-hooks.tsx

@@ -4,8 +4,8 @@ import type {
   VisibilityState,
 } from '@tanstack/react-table';
 import { parseAsInteger, useQueryState } from 'nuqs';
-import { useState } from 'react';
-import { useLocalStorage } from 'usehooks-ts';
+import { useEffect, useState } from 'react';
+import { useLocalStorage, useReadLocalStorage } from 'usehooks-ts';
 
 export const useDataTablePagination = (pageSize = 10) => {
   const [page, setPage] = useQueryState(
@@ -22,6 +22,12 @@ export const useDataTablePagination = (pageSize = 10) => {
   return { page, setPage, state };
 };
 
+export const useReadColumnVisibility = (persistentKey: string) => {
+  return useReadLocalStorage<Record<string, boolean>>(
+    `@op:${persistentKey}-column-visibility`,
+  );
+};
+
 export const useDataTableColumnVisibility = <TData,>(
   columns: ColumnDef<TData>[],
   persistentKey: string,
@@ -43,6 +49,13 @@ export const useDataTableColumnVisibility = <TData,>(
     }, {} as VisibilityState),
   );
 
+  // somewhat hack
+  // Set initial column visibility,
+  // otherwise will not useReadColumnVisibility be updated
+  useEffect(() => {
+    setColumnVisibility(columnVisibility);
+  }, []);
+
   const [columnOrder, setColumnOrder] = useLocalStorage<string[]>(
     `@op:${persistentKey}-column-order`,
     columns.map((column) => column.id!),

apps/start/src/modals/event-details.tsx

@@ -18,14 +18,8 @@ import { useTRPC } from '@/integrations/trpc/react';
 import { cn } from '@/utils/cn';
 import { getProfileName } from '@/utils/getters';
 import type { IClickhouseEvent, IServiceEvent } from '@openpanel/db';
-import { useQuery, useSuspenseQuery } from '@tanstack/react-query';
-import {
-  ArrowLeftIcon,
-  ArrowRightIcon,
-  FilterIcon,
-  Loader2Icon,
-  XIcon,
-} from 'lucide-react';
+import { useQuery } from '@tanstack/react-query';
+import { FilterIcon, XIcon } from 'lucide-react';
 import { omit } from 'ramda';
 import { useState } from 'react';
 import { popModal } from '.';

apps/start/src/routes/_app.$organizationId.$projectId_.events._tabs.conversions.tsx

@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import { useTRPC } from '@/integrations/trpc/react';
 import { useInfiniteQuery } from '@tanstack/react-query';
 import { createFileRoute } from '@tanstack/react-router';
@@ -18,12 +19,14 @@ function Component() {
     parseAsIsoDateTime,
   );
   const [endDate, setEndDate] = useQueryState('endDate', parseAsIsoDateTime);
+  const columnVisibility = useReadColumnVisibility('events');
   const query = useInfiniteQuery(
     trpc.event.conversions.infiniteQueryOptions(
       {
         projectId,
         startDate: startDate || undefined,
         endDate: endDate || undefined,
+        columnVisibility: columnVisibility ?? {},
       },
       {
         getNextPageParam: (lastPage) => lastPage.meta.next,

apps/start/src/routes/_app.$organizationId.$projectId_.events._tabs.events.tsx

@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
   useEventQueryFilters,
   useEventQueryNamesFilter,
@@ -21,6 +22,8 @@ function Component() {
   const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
   const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
   const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
+
   const query = useInfiniteQuery(
     trpc.event.events.infiniteQueryOptions(
       {
@@ -30,8 +33,10 @@ function Component() {
         profileId: '',
         startDate: startDate || undefined,
         endDate: endDate || undefined,
+        columnVisibility: columnVisibility ?? {},
       },
       {
+        enabled: columnVisibility !== null,
         getNextPageParam: (lastPage) => lastPage.meta.next,
       },
     ),

apps/start/src/routes/_app.$organizationId.$projectId_.profiles.$profileId._tabs.events.tsx

@@ -1,4 +1,5 @@
 import { EventsTable } from '@/components/events/table';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
   useEventQueryFilters,
   useEventQueryNamesFilter,
@@ -21,6 +22,7 @@ function Component() {
   const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
   const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
   const [eventNames] = useEventQueryNamesFilter();
+  const columnVisibility = useReadColumnVisibility('events');
   const query = useInfiniteQuery(
     trpc.event.events.infiniteQueryOptions(
       {
@@ -30,8 +32,10 @@ function Component() {
         startDate: startDate || undefined,
         endDate: endDate || undefined,
         events: eventNames,
+        columnVisibility: columnVisibility ?? {},
       },
       {
+        enabled: columnVisibility !== null,
         getNextPageParam: (lastPage) => lastPage.meta.next,
       },
     ),

apps/start/src/routes/_app.$organizationId.$projectId_.sessions_.$sessionId.tsx

@@ -3,12 +3,11 @@ import FullPageLoadingState from '@/components/full-page-loading-state';
 import { PageContainer } from '@/components/page-container';
 import { PageHeader } from '@/components/page-header';
 import { SerieIcon } from '@/components/report-chart/common/serie-icon';
-import { useDataTablePagination } from '@/components/ui/data-table/data-table-hooks';
+import { useReadColumnVisibility } from '@/components/ui/data-table/data-table-hooks';
 import {
   useEventQueryFilters,
   useEventQueryNamesFilter,
 } from '@/hooks/use-event-query-filters';
-import { useSearchQueryState } from '@/hooks/use-search-query-state';
 import { useTRPC } from '@/integrations/trpc/react';
 import { createProjectTitle } from '@/utils/title';
 import { useInfiniteQuery, useSuspenseQuery } from '@tanstack/react-query';
@@ -46,8 +45,6 @@ function Component() {
   const trpc = useTRPC();
 
   const LIMIT = 50;
-  const { page } = useDataTablePagination(LIMIT);
-  const { debouncedSearch } = useSearchQueryState();
 
   const { data: session } = useSuspenseQuery(
     trpc.session.byId.queryOptions({
@@ -60,7 +57,7 @@ function Component() {
   const [startDate] = useQueryState('startDate', parseAsIsoDateTime);
   const [endDate] = useQueryState('endDate', parseAsIsoDateTime);
   const [eventNames] = useEventQueryNamesFilter();
-
+  const columnVisibility = useReadColumnVisibility('events');
   const query = useInfiniteQuery(
     trpc.event.events.infiniteQueryOptions(
       {
@@ -70,8 +67,10 @@ function Component() {
         events: eventNames,
         startDate: startDate || undefined,
         endDate: endDate || undefined,
+        columnVisibility: columnVisibility ?? {},
       },
       {
+        enabled: columnVisibility !== null,
         getNextPageParam: (lastPage) => lastPage.meta.next,
       },
     ),

apps/worker/src/jobs/cron.delete-projects.ts

@@ -54,7 +54,7 @@ export async function deleteProjects(job: Job<CronQueuePayload>) {
     await ch.command({
       query,
       clickhouse_settings: {
-        lightweight_deletes_sync: 0,
+        lightweight_deletes_sync: '0',
       },
     });
   }