UBERF-9724: Github use of updated accounts

.vscode/launch.json

@@ -424,7 +424,7 @@
       "name": "Debug tool upgrade",
       "type": "node",
       "request": "launch",
-      "args": ["src/__start.ts", "upgrade-workspace", "mongo-1000-1"],
+      "args": ["src/__start.ts", "migrate-github-account", "--region", "cockroach", "--db", "%github"],
       "env": {
         "SERVER_SECRET": "secret",
         "MINIO_ACCESS_KEY": "minioadmin",
@@ -433,7 +433,7 @@
         "TRANSACTOR_URL": "ws://localhost:3333",
         "MONGO_URL": "mongodb://localhost:27017",
         "DB_URL": "mongodb://localhost:27017",
-        "ACCOUNTS_URL": "http://localhost:3000",
+        "ACCOUNTS_URL": "http://127.0.0.1:3000",
         "ACCOUNT_DB_URL": "mongodb://localhost:27017",
         "TELEGRAM_DATABASE": "telegram-service",
         "REKONI_URL": "http://localhost:4004",
@@ -565,21 +565,21 @@
       "request": "launch",
       "args": ["src/index.ts"],
       "env": {
-        "MONGO_URL": "mongodb://localhost:27018",
+        "MONGO_URL": "mongodb://localhost:27017",
         "SERVER_SECRET": "secret",
-        "ACCOUNTS_URL": "http://localhost:3003",
+        "ACCOUNTS_URL": "http://localhost:3000",
         "APP_ID": "${env:POD_GITHUB_APPID}",
         "CLIENT_ID": "${env:POD_GITHUB_CLIENTID}",
         "CLIENT_SECRET": "${env:POD_GITHUB_CLIENT_SECRET}",
         "PRIVATE_KEY": "${env:POD_GITHUB_PRIVATE_KEY}",
-        "COLLABORATOR_URL": "ws://huly.local:3079",
+        "COLLABORATOR_URL": "ws://huly.local:3078",
         "MINIO_ENDPOINT": "localhost",
         "MINIO_ACCESS_KEY": "minioadmin",
         "MINIO_SECRET_KEY": "minioadmin",
         "PLATFORM_OPERATION_LOGGING": "true",
         "FRONT_URL": "http://localhost:8080",
         "PORT": "3500",
-        "STATS_URL": "http://huly.local:4901"
+        "STATS_URL": "http://huly.local:4900"
       },
       "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
       "sourceMaps": true,

dev/tool/src/github.ts

@@ -0,0 +1,195 @@
+import core, {
+  buildSocialIdString,
+  DOMAIN_MODEL_TX,
+  systemAccountUuid,
+  TxProcessor,
+  type BackupClient,
+  type Client,
+  type Doc,
+  type Ref,
+  type TxCUD,
+  type WorkspaceUuid
+} from '@hcengineering/core'
+import { getAccountsFromTxes, getSocialKeyByOldEmail } from '@hcengineering/model-core'
+import { createClient, getAccountClient, getTransactorEndpoint } from '@hcengineering/server-client'
+import { generateToken } from '@hcengineering/server-token'
+import type { Db } from 'mongodb'
+
+/**
+ * @public
+ */
+export interface GithubIntegrationRecord {
+  installationId: number
+  workspace: string
+  accountId: string // Ref<Account>
+}
+
+/**
+ * @public
+ */
+export interface GithubUserRecord {
+  _id: string // login
+  code?: string | null
+  token?: string
+  expiresIn?: number | null // seconds
+  refreshToken?: string | null
+  refreshTokenExpiresIn?: number | null
+  authorized?: boolean
+  state?: string
+  scope?: string
+  error?: string | null
+
+  accounts: Record<string, string /* Ref<Account> */>
+}
+
+export async function performGithubAccountMigrations (db: Db, region: string | null): Promise<void> {
+  const token = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'admin', admin: 'true' })
+  const githubToken = generateToken(systemAccountUuid, '' as WorkspaceUuid, { service: 'github' })
+  const accountClient = getAccountClient(token)
+
+  const githubAccountClient = getAccountClient(githubToken)
+
+  const usersCollection = db.collection<GithubUserRecord>('users')
+
+  const integrationCollection = db.collection<GithubIntegrationRecord>('installations')
+
+  const integrations = await integrationCollection.find({}).toArray()
+  // Check and apply migrations
+  // We need to update all workspace information accordingly
+
+  const allWorkpaces = await accountClient.listWorkspaces(region)
+  const byId = new Map(allWorkpaces.map((it) => [it.uuid, it]))
+  const oldNewIds = new Map(allWorkpaces.map((it) => [it.dataId ?? it.uuid, it]))
+
+  const allAuthorizations = await usersCollection.find({}).toArray()
+
+  const wsToAuth = new Map<WorkspaceUuid, GithubUserRecord[]>()
+
+  for (const it of allAuthorizations) {
+    for (const ws of Object.keys(it.accounts)) {
+      const wsId = oldNewIds.get(ws as WorkspaceUuid) ?? byId.get(ws as WorkspaceUuid)
+      if (wsId !== undefined) {
+        wsToAuth.set(wsId.uuid, (wsToAuth.get(wsId.uuid) ?? []).concat(it))
+      }
+    }
+  }
+  const processed = new Set<string>()
+
+  const replaces = new Map<string, WorkspaceUuid>()
+  for (const it of integrations) {
+    const ws = oldNewIds.get(it.workspace as any) ?? byId.get(it.workspace as any)
+    if (ws != null) {
+      // Need to connect to workspace to get account mapping
+
+      it.workspace = ws.uuid
+      replaces.set(it.workspace, ws.uuid)
+
+      const wsToken = generateToken(systemAccountUuid, ws.uuid, { service: 'github', mode: 'backup' })
+      const endpoint = await getTransactorEndpoint(wsToken, 'external')
+      const client = (await createClient(endpoint, wsToken)) as BackupClient & Client
+
+      const systemAccounts = [core.account.System, core.account.ConfigUser]
+      const accountsTxes: TxCUD<Doc>[] = []
+
+      let idx: number | undefined
+
+      while (true) {
+        const info = await client.loadChunk(DOMAIN_MODEL_TX, idx)
+        idx = info.idx
+        const ids = Array.from(info.docs.map((it) => it.id as Ref<Doc>))
+        const docs = (await client.loadDocs(DOMAIN_MODEL_TX, ids)).filter((it) =>
+          TxProcessor.isExtendsCUD(it._class)
+        ) as TxCUD<Doc>[]
+        accountsTxes.push(...docs)
+        if (info.finished && idx !== undefined) {
+          await client.closeChunk(info.idx)
+          break
+        }
+      }
+      await client.close()
+
+      // await client.loadChunk(DOMAIN_MODEL_TX,  {
+      //   objectClass: { $in: ['core:class:Account', 'contact:class:PersonAccount'] as Ref<Class<Doc>>[] }
+      // })
+      const accounts: (Doc & { email?: string })[] = getAccountsFromTxes(accountsTxes)
+
+      const socialKeyByAccount: Record<string, string> = {}
+      for (const account of accounts) {
+        if (account.email === undefined) {
+          continue
+        }
+
+        if (systemAccounts.includes(account._id as any)) {
+          ;(socialKeyByAccount as any)[account._id] = account._id
+        } else {
+          socialKeyByAccount[account._id] = buildSocialIdString(getSocialKeyByOldEmail(account.email)) as any
+        }
+      }
+
+      const sid = socialKeyByAccount[it.accountId]
+
+      const person = sid !== undefined ? await accountClient.findSocialIdBySocialKey(sid) : undefined
+      if (person !== undefined) {
+        // Check/create integeration in account
+
+        const existing = await githubAccountClient.getIntegration({
+          kind: 'github',
+          workspaceUuid: ws?.uuid,
+          socialId: person
+        })
+
+        if (existing == null) {
+          await githubAccountClient.createIntegration({
+            kind: 'github',
+            workspaceUuid: ws?.uuid,
+            socialId: person,
+            data: {
+              installationId: it.installationId
+            }
+          })
+        }
+      }
+
+      const users = wsToAuth.get(ws.uuid)
+      for (const u of users ?? []) {
+        if (processed.has(u._id)) {
+          continue
+        }
+        processed.add(u._id)
+
+        const sid = socialKeyByAccount[u.accounts[ws.dataId ?? ws.uuid]]
+        if (sid !== undefined) {
+          const person = await accountClient.findSocialIdBySocialKey(sid)
+          if (person !== undefined) {
+            const { _id, accounts, ...data } = u
+
+            const existing = await githubAccountClient.getIntegration({
+              kind: 'github-user',
+              workspaceUuid: null,
+              socialId: person
+            })
+
+            if (existing == null) {
+              await githubAccountClient.createIntegration({
+                kind: 'github-user',
+                workspaceUuid: null,
+                socialId: person,
+                data: {
+                  login: u._id
+                }
+              })
+              // Check/create integeration in account
+              await githubAccountClient.addIntegrationSecret({
+                kind: 'github-user',
+                workspaceUuid: null,
+                socialId: person,
+                key: u._id, // github login
+                secret: JSON.stringify(data)
+              })
+            }
+          }
+        }
+      }
+    }
+  }
+}

dev/tool/src/index.ts

@@ -71,6 +71,7 @@ import {
   createMongoAdapter,
   createMongoDestroyAdapter,
   createMongoTxAdapter,
+  getMongoClient,
   shutdownMongo
 } from '@hcengineering/mongo'
 import { backupDownload } from '@hcengineering/server-backup/src/backup'
@@ -93,6 +94,7 @@ import { getAccountDBUrl, getMongoDBUrl } from './__start'
 import { changeConfiguration } from './configuration'
 
 import { moveAccountDbFromMongoToPG } from './db'
+import { performGithubAccountMigrations } from './github'
 import { getToolToken, getWorkspace, getWorkspaceTransactorEndpoint } from './utils'
 
 const colorConstants = {
@@ -2399,6 +2401,20 @@ export function devTool (
   //     })
   //   })
 
+  program
+    .command('migrate-github-account')
+    .option('--db <db>', 'Github DB', '%github')
+    .option('--region <region>', 'Github DB')
+    .action(async (cmd: { db: string, region?: string }) => {
+      const mongodbUri = getMongoDBUrl()
+      const client = getMongoClient(mongodbUri)
+      const _client = await client.getClient()
+
+      await performGithubAccountMigrations(_client.db(cmd.db), cmd.region ?? null)
+      await _client.close()
+      client.close()
+    })
+
   program
     .command('queue-init-topics')
     .description('create required kafka topics')

models/core/src/migration.ts

@@ -316,7 +316,7 @@ export async function getSocialKeyByOldAccount (client: MigrationClient): Promis
   })
   const accounts = getAccountsFromTxes(accountsTxes)
 
-  const socialKeyByAccount: Record<string, PersonId> = {}
+  const socialKeyByAccount: Record<string, string> = {}
   for (const account of accounts) {
     if (account.email === undefined) {
       continue

packages/account-client/src/client.ts

@@ -169,12 +169,12 @@ export interface AccountClient {
 }
 
 /** @public */
-export function getClient (accountsUrl?: string, token?: string): AccountClient {
+export function getClient (accountsUrl?: string, token?: string, retryTimeoutMs?: number): AccountClient {
   if (accountsUrl === undefined) {
     throw new Error('Accounts url not specified')
   }
 
-  return new AccountClientImpl(accountsUrl, token)
+  return new AccountClientImpl(accountsUrl, token, retryTimeoutMs)
 }
 
 interface Request {
@@ -188,7 +188,8 @@ class AccountClientImpl implements AccountClient {
 
   constructor (
     private readonly url: string,
-    private readonly token?: string
+    private readonly token?: string,
+    retryTimeoutMs?: number
   ) {
     if (url === '') {
       throw new Error('Accounts url not specified')
@@ -207,7 +208,7 @@ class AccountClientImpl implements AccountClient {
       },
       ...(isBrowser ? { credentials: 'include' } : {})
     }
-    this.rpc = withRetryUntilTimeout(this._rpc.bind(this))
+    this.rpc = withRetryUntilTimeout(this._rpc.bind(this), retryTimeoutMs ?? 5000)
   }
 
   async getProviders (): Promise<string[]> {
@@ -907,7 +908,7 @@ class AccountClientImpl implements AccountClient {
 function withRetry<T, F extends (...args: any[]) => Promise<T>> (
   f: F,
   shouldFail: (err: any, attempt: number) => boolean,
-  intervalMs: number = 1000
+  intervalMs: number = 25
 ): F {
   return async function (...params: any[]): Promise<T> {
     let attempt = 0
@@ -921,6 +922,9 @@ function withRetry<T, F extends (...args: any[]) => Promise<T>> (
 
         attempt++
         await new Promise<void>((resolve) => setTimeout(resolve, intervalMs))
+        if (intervalMs < 1000) {
+          intervalMs += 100
+        }
       }
     }
   } as F

plugins/activity-resources/src/components/activity-message/ActivityMessageTemplate.svelte

@@ -14,36 +14,37 @@
 -->
 <script lang="ts">
   import activity, {
+    ActivityMessage,
     ActivityMessageViewlet,
-    DisplayActivityMessage,
     ActivityMessageViewType,
-    ActivityMessage
+    DisplayActivityMessage
   } from '@hcengineering/activity'
   import { Person } from '@hcengineering/contact'
   import { Avatar, SystemAvatar } from '@hcengineering/contact-resources'
-  import core, { Ref } from '@hcengineering/core'
+  import core, { Ref, type SocialId } from '@hcengineering/core'
+  import notification from '@hcengineering/notification'
+  import { Asset } from '@hcengineering/platform'
   import { ComponentExtensions, getClient } from '@hcengineering/presentation'
   import { Action, Icon, Label } from '@hcengineering/ui'
-  import { getActions, restrictionStore, showMenu } from '@hcengineering/view-resources'
-  import { Asset } from '@hcengineering/platform'
   import { Action as ViewAction } from '@hcengineering/view'
-  import notification from '@hcengineering/notification'
+  import { getActions, restrictionStore, showMenu } from '@hcengineering/view-resources'
 
-  import ReactionsPresenter from '../reactions/ReactionsPresenter.svelte'
-  import ActivityMessagePresenter from './ActivityMessagePresenter.svelte'
-  import ActivityMessageActions from '../ActivityMessageActions.svelte'
-  import { isReactionMessage } from '../../activityMessagesUtils'
   import { savedMessagesStore } from '../../activity'
+  import { isReactionMessage } from '../../activityMessagesUtils'
+  import { MessageInlineAction } from '../../types'
+  import ActivityMessageActions from '../ActivityMessageActions.svelte'
   import MessageTimestamp from '../MessageTimestamp.svelte'
+  import ReactionsPresenter from '../reactions/ReactionsPresenter.svelte'
   import Replies from '../Replies.svelte'
-  import { MessageInlineAction } from '../../types'
+  import ActivityMessagePresenter from './ActivityMessagePresenter.svelte'
   import InlineAction from './InlineAction.svelte'
 
   export let message: DisplayActivityMessage
   export let parentMessage: DisplayActivityMessage | undefined = undefined
 
   export let viewlet: ActivityMessageViewlet | undefined = undefined
   export let person: Person | undefined = undefined
+  export let socialId: SocialId | undefined = undefined
   export let actions: Action[] = []
   export let showNotify: boolean = false
   export let isHighlighted: boolean = false
@@ -219,6 +220,9 @@
               <div class="username">
                 <ComponentExtensions extension={activity.extension.ActivityEmployeePresenter} props={{ person }} />
               </div>
+              {#if socialId !== undefined}
+                ({socialId.type})
+              {/if}
             {:else}
               <div class="strong">
                 <Label label={core.string.System} />