From 367f5d5ef3d58f007ac52c89c79d48a7eaeefde0 Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 00:38:29 +0200 Subject: [PATCH 01/10] Local ClickHouse cluster configuration --- .docker/clickhouse/cluster/config1.xml | 85 ++++++++++++++++++++++++++ .docker/clickhouse/cluster/config2.xml | 85 ++++++++++++++++++++++++++ .docker/clickhouse/cluster/macros1.xml | 7 +++ .docker/clickhouse/cluster/macros2.xml | 7 +++ .docker/clickhouse/cluster/users.xml | 33 ++++++++++ .docker/nginx/local.conf | 11 ++++ __tests__/integration/insert.test.ts | 26 +++++--- docker-compose.cluster.yml | 46 ++++++++++++++ docker-compose.yml | 2 +- 9 files changed, 292 insertions(+), 10 deletions(-) create mode 100644 .docker/clickhouse/cluster/config1.xml create mode 100644 .docker/clickhouse/cluster/config2.xml create mode 100644 .docker/clickhouse/cluster/macros1.xml create mode 100644 .docker/clickhouse/cluster/macros2.xml create mode 100644 .docker/clickhouse/cluster/users.xml create mode 100644 .docker/nginx/local.conf create mode 100644 docker-compose.cluster.yml diff --git a/.docker/clickhouse/cluster/config1.xml b/.docker/clickhouse/cluster/config1.xml new file mode 100644 index 00000000..8512e36b --- /dev/null +++ b/.docker/clickhouse/cluster/config1.xml @@ -0,0 +1,85 @@ + + + + 8123 + 9009 + clickhouse1 + + users.xml + default + default + + 5368709120 + + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + /var/lib/clickhouse/access/ + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + 1 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + trace + 10000 + + + + + 1 + clickhouse1 + 9000 + + + 2 + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9181 + + + clickhouse2 + 9181 + + + + + /clickhouse/test_cluster/task_queue/ddl + + + \ No newline at end of file diff --git a/.docker/clickhouse/cluster/config2.xml b/.docker/clickhouse/cluster/config2.xml new file mode 100644 index 00000000..07800ee6 --- /dev/null +++ b/.docker/clickhouse/cluster/config2.xml @@ -0,0 +1,85 @@ + + + + 8123 + 9009 + clickhouse2 + + users.xml + default + default + + 5368709120 + + /var/lib/clickhouse/ + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + /var/lib/clickhouse/access/ + + + debug + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + 1 + + + + + + + clickhouse1 + 9000 + + + clickhouse2 + 9000 + + + + + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + trace + 10000 + + + + + 1 + clickhouse1 + 9000 + + + 2 + clickhouse2 + 9000 + + + + + + + clickhouse1 + 9181 + + + clickhouse2 + 9181 + + + + + /clickhouse/test_cluster/task_queue/ddl + + + \ No newline at end of file diff --git a/.docker/clickhouse/cluster/macros1.xml b/.docker/clickhouse/cluster/macros1.xml new file mode 100644 index 00000000..70de6284 --- /dev/null +++ b/.docker/clickhouse/cluster/macros1.xml @@ -0,0 +1,7 @@ + + + test_cluster + clickhouse1 + 1 + + \ No newline at end of file diff --git a/.docker/clickhouse/cluster/macros2.xml b/.docker/clickhouse/cluster/macros2.xml new file mode 100644 index 00000000..c640d0ec --- /dev/null +++ b/.docker/clickhouse/cluster/macros2.xml @@ -0,0 +1,7 @@ + + + test_cluster + clickhouse2 + 1 + + \ No newline at end of file diff --git a/.docker/clickhouse/cluster/users.xml b/.docker/clickhouse/cluster/users.xml new file mode 100644 index 00000000..521dc043 --- /dev/null +++ b/.docker/clickhouse/cluster/users.xml @@ -0,0 +1,33 @@ + + + + + + random + + + + + + + + ::/0 + + default + default + + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + \ No newline at end of file diff --git a/.docker/nginx/local.conf b/.docker/nginx/local.conf new file mode 100644 index 00000000..cff89fa6 --- /dev/null +++ b/.docker/nginx/local.conf @@ -0,0 +1,11 @@ +upstream clickhouse_cluster { + server clickhouse1:8123; + server clickhouse2:8123; +} + +server { + listen 8123; + location / { + proxy_pass http://clickhouse_cluster; + } +} \ No newline at end of file diff --git a/__tests__/integration/insert.test.ts b/__tests__/integration/insert.test.ts index 2f3e4a10..2c0d185a 100644 --- a/__tests__/integration/insert.test.ts +++ b/__tests__/integration/insert.test.ts @@ -7,16 +7,24 @@ describe('insert', () => { let client: ClickHouseClient; let tableName: string; beforeEach(async () => { - client = await createTestClient(); - tableName = `test_table_${guid()}`; - await createTable(client, (engine) => { - return ` - CREATE TABLE ${tableName} - (id UInt64, name String, sku Array(UInt8)) - ${engine} - ORDER BY (id) - `; + client = await createTestClient({ + clickhouse_settings: { + insert_quorum: 2, + }, }); + tableName = `test_table_${guid()}`; + await createTable( + client, + (engine) => { + return ` + CREATE TABLE ${tableName} ON CLUSTER '{cluster}' + (id UInt64, name String, sku Array(UInt8)) + ${engine} + ORDER BY (id) + `; + }, + `ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}')` + ); }); afterEach(async () => { await client.close(); diff --git a/docker-compose.cluster.yml b/docker-compose.cluster.yml new file mode 100644 index 00000000..b7449803 --- /dev/null +++ b/docker-compose.cluster.yml @@ -0,0 +1,46 @@ +version: '2.3' + +services: + clickhouse1: + image: 'clickhouse/clickhouse-server:22.6.5.22-alpine' + ulimits: + nofile: + soft: 262144 + hard: 262144 + hostname: clickhouse1 + container_name: clickhouse-js-clickhouse-server-node-1 + ports: + - '8124:8123' + - '9000:9000' + - '9181:9181' + volumes: + - './.docker/clickhouse/cluster/config1.xml:/etc/clickhouse-server/config.xml' + - './.docker/clickhouse/cluster/macros1.xml:/etc/clickhouse-server/config.d/macros.xml' + - './.docker/clickhouse/cluster/users.xml:/etc/clickhouse-server/users.xml' + + clickhouse2: + image: 'clickhouse/clickhouse-server:22.6.5.22-alpine' + ulimits: + nofile: + soft: 262144 + hard: 262144 + hostname: clickhouse2 + container_name: clickhouse-js-clickhouse-server-node-2 + ports: + - '8125:8123' + - '9001:9000' + - '9182:9181' + volumes: + - './.docker/clickhouse/cluster/config2.xml:/etc/clickhouse-server/config.xml' + - './.docker/clickhouse/cluster/macros2.xml:/etc/clickhouse-server/config.d/macros.xml' + - './.docker/clickhouse/cluster/users.xml:/etc/clickhouse-server/users.xml' + + # Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests + nginx: + image: 'nginx:1.23.1-alpine' + hostname: nginx + ports: + - '8123:8123' + volumes: + - './.docker/nginx/local.conf:/etc/nginx/conf.d/local.conf' + container_name: clickhouse-js-nginx \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 25d574b1..a261033d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: clickhouse: - image: 'clickhouse/clickhouse-server:22.3.10.22-alpine' + image: 'clickhouse/clickhouse-server:22.6.5.22-alpine' container_name: 'clickhouse-js-clickhouse-server' ports: - '8123:8123' From 88a5c805caed222b9b570017f28c52bf4825fac9 Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 14:24:58 +0200 Subject: [PATCH 02/10] Log base http adapter commands --- src/connection/adapter/base_http_adapter.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/connection/adapter/base_http_adapter.ts b/src/connection/adapter/base_http_adapter.ts index 769f03e4..53479b77 100644 --- a/src/connection/adapter/base_http_adapter.ts +++ b/src/connection/adapter/base_http_adapter.ts @@ -15,6 +15,7 @@ import type { import { toSearchParams } from './http_search_params'; import { transformUrl } from './transform_url'; import { getAsText, isStream } from '../../utils'; +import { Rows } from '../../result'; export interface RequestParams { method: 'GET' | 'POST'; @@ -258,13 +259,16 @@ export abstract class BaseHttpAdapter implements Connection { params.query_params ); - await this.request({ + const stream = await this.request({ method: 'POST', url: transformUrl({ url: this.config.host, pathname: '/', searchParams }), body: params.query, abort_signal: params.abort_signal, }); + const rows = new Rows(stream, 'TabSeparated'); + const text = await rows.text(); + console.log(`Server returned:\n${text}`); // return await getAsText(result); } From 9d2026278fdb77da82af052130aa8133787beb5c Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 14:27:30 +0200 Subject: [PATCH 03/10] Log base http adapter commands --- src/connection/adapter/base_http_adapter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connection/adapter/base_http_adapter.ts b/src/connection/adapter/base_http_adapter.ts index 53479b77..b81c410e 100644 --- a/src/connection/adapter/base_http_adapter.ts +++ b/src/connection/adapter/base_http_adapter.ts @@ -268,7 +268,7 @@ export abstract class BaseHttpAdapter implements Connection { const rows = new Rows(stream, 'TabSeparated'); const text = await rows.text(); - console.log(`Server returned:\n${text}`); + console.log(`Command returned:\n${text}`); // return await getAsText(result); } From d3179bd3906d7095fb3d1ee66b5961f631117d6d Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 14:28:10 +0200 Subject: [PATCH 04/10] Fix GHA --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c1e6c5d0..a73583ad 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -5,8 +5,8 @@ on: branches: - main pull_request: - branches: - - main +# branches: +# - main schedule: - cron: '0 9 * * *' From 8f23c3e7feb911c64906899efe175db08559fad5 Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 14:29:46 +0200 Subject: [PATCH 05/10] Fix GHA --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a73583ad..f25c2a90 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -5,8 +5,8 @@ on: branches: - main pull_request: -# branches: -# - main + branches: + - '*' schedule: - cron: '0 9 * * *' From 79f8a5a4746dbaad2123d0853873f6da67c91f19 Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 14:30:34 +0200 Subject: [PATCH 06/10] Fix GHA --- .github/workflows/run-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f25c2a90..605823c9 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -3,7 +3,7 @@ name: 'run tests' on: push: branches: - - main + - '*' pull_request: branches: - '*' From dc29eec9e0622f6b5202f725232d987338dfd01f Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 15:37:12 +0200 Subject: [PATCH 07/10] Revert GHA --- .github/workflows/run-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 605823c9..c1e6c5d0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -3,10 +3,10 @@ name: 'run tests' on: push: branches: - - '*' + - main pull_request: branches: - - '*' + - main schedule: - cron: '0 9 * * *' From 97db905e487be69ecc472d91a4b2230fe50654cb Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 16:29:07 +0200 Subject: [PATCH 08/10] Fix `command` and related tests, use more flexible test env, rename configs --- .../{config1.xml => server1_config.xml} | 0 .../{macros1.xml => server1_macros.xml} | 0 .../{config2.xml => server2_config.xml} | 0 .../{macros2.xml => server2_macros.xml} | 0 .github/workflows/run-tests.yml | 2 +- __tests__/integration/command.test.ts | 90 ++++++++++++++----- __tests__/integration/insert.test.ts | 39 +++++--- __tests__/utils/client.ts | 50 ++++++++--- docker-compose.cluster.yml | 8 +- src/client.ts | 16 +++- src/connection/adapter/base_http_adapter.ts | 10 +-- src/connection/connection.ts | 2 +- 12 files changed, 153 insertions(+), 64 deletions(-) rename .docker/clickhouse/cluster/{config1.xml => server1_config.xml} (100%) rename .docker/clickhouse/cluster/{macros1.xml => server1_macros.xml} (100%) rename .docker/clickhouse/cluster/{config2.xml => server2_config.xml} (100%) rename .docker/clickhouse/cluster/{macros2.xml => server2_macros.xml} (100%) diff --git a/.docker/clickhouse/cluster/config1.xml b/.docker/clickhouse/cluster/server1_config.xml similarity index 100% rename from .docker/clickhouse/cluster/config1.xml rename to .docker/clickhouse/cluster/server1_config.xml diff --git a/.docker/clickhouse/cluster/macros1.xml b/.docker/clickhouse/cluster/server1_macros.xml similarity index 100% rename from .docker/clickhouse/cluster/macros1.xml rename to .docker/clickhouse/cluster/server1_macros.xml diff --git a/.docker/clickhouse/cluster/config2.xml b/.docker/clickhouse/cluster/server2_config.xml similarity index 100% rename from .docker/clickhouse/cluster/config2.xml rename to .docker/clickhouse/cluster/server2_config.xml diff --git a/.docker/clickhouse/cluster/macros2.xml b/.docker/clickhouse/cluster/server2_macros.xml similarity index 100% rename from .docker/clickhouse/cluster/macros2.xml rename to .docker/clickhouse/cluster/server2_macros.xml diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c1e6c5d0..ce275b52 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -52,6 +52,6 @@ jobs: CLICKHOUSE_CLOUD_HOST: ${{ secrets.CLICKHOUSE_CLOUD_HOST }} CLICKHOUSE_CLOUD_USERNAME: ${{ secrets.CLICKHOUSE_CLOUD_USERNAME }} CLICKHOUSE_CLOUD_PASSWORD: ${{ secrets.CLICKHOUSE_CLOUD_PASSWORD }} - CLICKHOUSE_CLOUD_ENABLED: 'true' + CLICKHOUSE_TEST_ENVIRONMENT: 'cloud' run: | npm test diff --git a/__tests__/integration/command.test.ts b/__tests__/integration/command.test.ts index 302759e6..12d5586f 100644 --- a/__tests__/integration/command.test.ts +++ b/__tests__/integration/command.test.ts @@ -1,43 +1,49 @@ import { expect } from 'chai'; -import { createClient, type ClickHouseClient } from '../../src'; -import type { ResponseJSON } from '../../src/clickhouse_types'; +import type { ResponseJSON } from '../../src'; +import { type ClickHouseClient } from '../../src'; +import { + createTestClient, + getClickHouseTestEnvironment, + TestEnv, +} from '../utils/client'; +import { guid } from '../utils'; describe('command', () => { let client: ClickHouseClient; + beforeEach(() => { + client = createTestClient(); + }); afterEach(async () => { - await client.command({ query: 'DROP TABLE example' }); await client.close(); }); it('sends a command to execute', async () => { - client = createClient(); - const ddl = - 'CREATE TABLE example (id UInt64, name String, sku Array(UInt8), timestamp DateTime) Engine = Memory'; + const { ddl, tableName, engine } = getDDL(); - await client.command({ query: ddl }); + const commandResult = await client.command({ + query: ddl, + format: 'TabSeparated', + }); + await commandResult.text(); - const result = await client.select({ - query: `SELECT * from system.tables where name = 'example'`, + const selectResult = await client.select({ + query: `SELECT * from system.tables where name = '${tableName}'`, format: 'JSON', }); - const { data, rows } = await result.json< + const { data, rows } = await selectResult.json< ResponseJSON<{ name: string; engine: string; create_table_query: string }> >(); expect(rows).to.equal(1); const table = data[0]; - expect(table.name).equal('example'); - expect(table.engine).equal('Memory'); + expect(table.name).equal(tableName); + expect(table.engine).equal(engine); expect(table.create_table_query).to.be.a('string'); }); it('does not swallow ClickHouse error', (done) => { - client = createClient(); - - const ddl = - 'CREATE TABLE example (id UInt64, name String, sku Array(UInt8), timestamp DateTime) Engine = Memory'; - + const { ddl, tableName } = getDDL(); Promise.resolve() .then(() => client.command({ query: ddl })) .then(() => client.command({ query: ddl })) @@ -45,21 +51,21 @@ describe('command', () => { expect(e.code).to.equal('57'); expect(e.type).to.equal('TABLE_ALREADY_EXISTS'); // TODO remove whitespace from end - expect(e.message).equal('Table default.example already exists. '); + expect(e.message).equal(`Table default.${tableName} already exists. `); done(); }); }); it.skip('can specify a parameterized query', async () => { - client = createClient(); - await client.command({ - query: - 'CREATE TABLE {table_name: String} (id UInt64, name String, sku Array(UInt8), timestamp DateTime) Engine = Memory', + const commandResult = await client.command({ + query: '', query_params: { table_name: 'example', }, }); + await commandResult.text(); + // FIXME: use different DDL based on the TestEnv const result = await client.select({ query: `SELECT * from system.tables where name = 'example'`, format: 'JSON', @@ -74,3 +80,43 @@ describe('command', () => { expect(table.name).to.equal('example'); }); }); + +function getDDL(): { + ddl: string; + tableName: string; + engine: string; +} { + const env = getClickHouseTestEnvironment(); + const tableName = `command_test_${guid()}`; + switch (env) { + // ENGINE can be omitted in the cloud statements: + // it will use ReplicatedMergeTree and will add ON CLUSTER as well + case TestEnv.Cloud: { + const ddl = ` + CREATE TABLE ${tableName} + (id UInt64, name String, sku Array(UInt8), timestamp DateTime) + ORDER BY (id) + `; + return { ddl, tableName, engine: 'ReplicatedMergeTree' }; + } + case TestEnv.LocalSingleNode: { + const ddl = ` + CREATE TABLE ${tableName} + (id UInt64, name String, sku Array(UInt8), timestamp DateTime) + ENGINE = MergeTree() + ORDER BY (id) + `; + return { ddl, tableName, engine: 'MergeTree' }; + } + + case TestEnv.LocalCluster: { + const ddl = ` + CREATE TABLE ${tableName} ON CLUSTER '{cluster}' + (id UInt64, name String, sku Array(UInt8), timestamp DateTime) + ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}') + ORDER BY (id) + `; + return { ddl, tableName, engine: 'ReplicatedMergeTree' }; + } + } +} diff --git a/__tests__/integration/insert.test.ts b/__tests__/integration/insert.test.ts index 2c0d185a..3701bdae 100644 --- a/__tests__/integration/insert.test.ts +++ b/__tests__/integration/insert.test.ts @@ -2,6 +2,7 @@ import { expect } from 'chai'; import type { ResponseJSON } from '../../src'; import { type ClickHouseClient } from '../../src'; import { createTable, createTestClient, guid } from '../utils'; +import { TestEnv } from '../utils/client'; describe('insert', () => { let client: ClickHouseClient; @@ -13,18 +14,32 @@ describe('insert', () => { }, }); tableName = `test_table_${guid()}`; - await createTable( - client, - (engine) => { - return ` - CREATE TABLE ${tableName} ON CLUSTER '{cluster}' - (id UInt64, name String, sku Array(UInt8)) - ${engine} - ORDER BY (id) - `; - }, - `ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}')` - ); + await createTable(client, (env) => { + switch (env) { + // ENGINE can be omitted in the cloud statements: + // it will use ReplicatedMergeTree and will add ON CLUSTER as well + case TestEnv.Cloud: + return ` + CREATE TABLE ${tableName} + (id UInt64, name String, sku Array(UInt8)) + ORDER BY (id) + `; + case TestEnv.LocalSingleNode: + return ` + CREATE TABLE ${tableName} + (id UInt64, name String, sku Array(UInt8)) + ENGINE MergeTree() + ORDER BY (id) + `; + case TestEnv.LocalCluster: + return ` + CREATE TABLE ${tableName} ON CLUSTER '{cluster}' + (id UInt64, name String, sku Array(UInt8)) + ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}') + ORDER BY (id) + `; + } + }); }); afterEach(async () => { await client.close(); diff --git a/__tests__/utils/client.ts b/__tests__/utils/client.ts index a67e33be..d9161d9f 100644 --- a/__tests__/utils/client.ts +++ b/__tests__/utils/client.ts @@ -1,26 +1,39 @@ import { ClickHouseClient, ClickHouseClientConfigOptions, + ClickHouseSettings, createClient, } from '../../src'; import { guid } from './guid'; +export enum TestEnv { + Cloud = 'CLOUD', + LocalSingleNode = 'LOCAL_SINGLE_NODE', + LocalCluster = 'LOCAL_CLUSTER', +} + export function createTestClient( - config?: ClickHouseClientConfigOptions + config: ClickHouseClientConfigOptions = {} ): ClickHouseClient { - if (isClickHouseCloudEnabled()) { + const env = getClickHouseTestEnvironment(); + const clickHouseSettings: ClickHouseSettings = {}; + if (env === TestEnv.LocalCluster || env === TestEnv.Cloud) { + clickHouseSettings.insert_quorum = 2; + } + if (env === TestEnv.Cloud) { console.log('Using ClickHouse Cloud client'); return createClient({ host: getFromEnv('CLICKHOUSE_CLOUD_HOST'), username: getFromEnv('CLICKHOUSE_CLOUD_USERNAME'), password: getFromEnv('CLICKHOUSE_CLOUD_PASSWORD'), - clickhouse_settings: { - insert_quorum: 2, - }, + ...clickHouseSettings, ...config, }); } else { - return createClient(config); + return createClient({ + ...clickHouseSettings, + ...config, + }); } } @@ -37,12 +50,10 @@ export async function createRandomDatabase( export async function createTable( client: ClickHouseClient, - definition: (engine: string) => string, - engine = 'MergeTree()' + definition: (environment: TestEnv) => string ) { - const ddl = isClickHouseCloudEnabled() - ? definition('') - : definition(`ENGINE ${engine}`); + const env = getClickHouseTestEnvironment(); + const ddl = definition(env); await client.command({ query: ddl, }); @@ -57,6 +68,19 @@ function getFromEnv(key: string): string { return value; } -function isClickHouseCloudEnabled() { - return process.env['CLICKHOUSE_CLOUD_ENABLED'] === 'true'; +export function getClickHouseTestEnvironment(): TestEnv { + let env; + switch (process.env['CLICKHOUSE_TEST_ENVIRONMENT']) { + case 'CLOUD': + env = TestEnv.Cloud; + break; + case 'LOCAL_CLUSTER': + env = TestEnv.LocalCluster; + break; + default: + env = TestEnv.LocalSingleNode; + break; + } + console.log(`Using ${env} test environment`); + return env; } diff --git a/docker-compose.cluster.yml b/docker-compose.cluster.yml index b7449803..a084c878 100644 --- a/docker-compose.cluster.yml +++ b/docker-compose.cluster.yml @@ -14,8 +14,8 @@ services: - '9000:9000' - '9181:9181' volumes: - - './.docker/clickhouse/cluster/config1.xml:/etc/clickhouse-server/config.xml' - - './.docker/clickhouse/cluster/macros1.xml:/etc/clickhouse-server/config.d/macros.xml' + - './.docker/clickhouse/cluster/server1_config.xml:/etc/clickhouse-server/config.xml' + - './.docker/clickhouse/cluster/server1_macros.xml:/etc/clickhouse-server/config.d/macros.xml' - './.docker/clickhouse/cluster/users.xml:/etc/clickhouse-server/users.xml' clickhouse2: @@ -31,8 +31,8 @@ services: - '9001:9000' - '9182:9181' volumes: - - './.docker/clickhouse/cluster/config2.xml:/etc/clickhouse-server/config.xml' - - './.docker/clickhouse/cluster/macros2.xml:/etc/clickhouse-server/config.d/macros.xml' + - './.docker/clickhouse/cluster/server2_config.xml:/etc/clickhouse-server/config.xml' + - './.docker/clickhouse/cluster/server2_macros.xml:/etc/clickhouse-server/config.d/macros.xml' - './.docker/clickhouse/cluster/users.xml:/etc/clickhouse-server/users.xml' # Using Nginx as a cluster entrypoint and a round-robin load balancer for HTTP requests diff --git a/src/client.ts b/src/client.ts index a5ad7110..120411e1 100644 --- a/src/client.ts +++ b/src/client.ts @@ -44,6 +44,7 @@ export interface SelectParams extends BaseParams { export interface CommandParams extends BaseParams { query: string; + format?: DataFormat; } export interface InsertParams extends BaseParams { @@ -140,13 +141,16 @@ export class ClickHouseClient { return new Rows(stream, format); } - async command(params: CommandParams): Promise { - const query = params.query.trim(); + async command(params: CommandParams): Promise { + const format = params.format ?? 'JSON'; + const query = formatCommandQuery(params.query, format); - await this.connection.command({ + const stream = await this.connection.command({ query, ...this.getBaseParams(params), }); + + return new Rows(stream, format); } async insert(params: InsertParams): Promise { @@ -184,6 +188,12 @@ function formatSelectQuery(query: string, format: DataFormat): string { return query + ' \nFORMAT ' + format; } +// it is a duplicate of `formatSelectQuery`, but it might differ in the future +function formatCommandQuery(query: string, format: DataFormat): string { + query = query.trim(); + return query + ' \nFORMAT ' + format; +} + function validateInsertValues( values: ReadonlyArray | Stream.Readable ): void { diff --git a/src/connection/adapter/base_http_adapter.ts b/src/connection/adapter/base_http_adapter.ts index b81c410e..80e5dde7 100644 --- a/src/connection/adapter/base_http_adapter.ts +++ b/src/connection/adapter/base_http_adapter.ts @@ -15,7 +15,6 @@ import type { import { toSearchParams } from './http_search_params'; import { transformUrl } from './transform_url'; import { getAsText, isStream } from '../../utils'; -import { Rows } from '../../result'; export interface RequestParams { method: 'GET' | 'POST'; @@ -253,23 +252,18 @@ export abstract class BaseHttpAdapter implements Connection { }); } - async command(params: BaseParams): Promise { + async command(params: BaseParams): Promise { const searchParams = toSearchParams( params.clickhouse_settings, params.query_params ); - const stream = await this.request({ + return await this.request({ method: 'POST', url: transformUrl({ url: this.config.host, pathname: '/', searchParams }), body: params.query, abort_signal: params.abort_signal, }); - - const rows = new Rows(stream, 'TabSeparated'); - const text = await rows.text(); - console.log(`Command returned:\n${text}`); - // return await getAsText(result); } async insert(params: InsertParams): Promise { diff --git a/src/connection/connection.ts b/src/connection/connection.ts index c2eee13b..1ed5bbd6 100644 --- a/src/connection/connection.ts +++ b/src/connection/connection.ts @@ -34,7 +34,7 @@ export interface Connection { ping(): Promise; close(): Promise; select(params: BaseParams): Promise; - command(params: BaseParams): Promise; + command(params: BaseParams): Promise; insert(params: InsertParams): Promise; } From e7bef46e8f82a24d747db164f5bb3b390d288d0f Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 16:57:08 +0200 Subject: [PATCH 09/10] Remove duplicate settings --- __tests__/integration/insert.test.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/__tests__/integration/insert.test.ts b/__tests__/integration/insert.test.ts index 3701bdae..bda5980e 100644 --- a/__tests__/integration/insert.test.ts +++ b/__tests__/integration/insert.test.ts @@ -8,11 +8,7 @@ describe('insert', () => { let client: ClickHouseClient; let tableName: string; beforeEach(async () => { - client = await createTestClient({ - clickhouse_settings: { - insert_quorum: 2, - }, - }); + client = await createTestClient(); tableName = `test_table_${guid()}`; await createTable(client, (env) => { switch (env) { From 7e0306ac1a81636db2707c9e4ae6e15d69db38a7 Mon Sep 17 00:00:00 2001 From: "serge.klochkov" Date: Wed, 10 Aug 2022 16:57:52 +0200 Subject: [PATCH 10/10] Lowercase test env variable --- __tests__/utils/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/__tests__/utils/client.ts b/__tests__/utils/client.ts index d9161d9f..bf9aa1b7 100644 --- a/__tests__/utils/client.ts +++ b/__tests__/utils/client.ts @@ -7,9 +7,9 @@ import { import { guid } from './guid'; export enum TestEnv { - Cloud = 'CLOUD', - LocalSingleNode = 'LOCAL_SINGLE_NODE', - LocalCluster = 'LOCAL_CLUSTER', + Cloud = 'cloud', + LocalSingleNode = 'local_single_node', + LocalCluster = 'local_cluster', } export function createTestClient(