Skip to content

Commit f327451

Browse files
richcoder221richcoder221
committed
first push
1 parent cdd90e4 commit f327451

File tree

7 files changed

+322
-0
lines changed

7 files changed

+322
-0
lines changed

‎README.md

+7
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,9 @@
11
# questdb-devops
22
Quick Deployment of QuestDB with Envoy on the Cloud using Terraform and Docker
3+
4+
5+
6+
1. install terraform and gcloud cli
7+
2. setup gcloud cli login
8+
3. in terraform folder run terraform apply
9+
4. run setup.sh script make sure the persistent volume on linux mahcine is mounted to /dev/sdb before use chmod +x to enable permission to run

‎questdb/docker-compose.yaml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
services:
2+
questdb:
3+
image: questdb/questdb:latest
4+
container_name: questdb
5+
restart: always
6+
volumes:
7+
- /questdb_zfs:/var/lib/questdb
8+
networks:
9+
- my_private
10+
environment:
11+
- QDB_LINE_DEFAULT_PARTITION_BY=YEAR
12+
- QDB_CAIRO_WAL_WRITER_DATA_APPEND_PAGE_SIZE=128K
13+
- QDB_CAIRO_WRITER_DATA_APPEND_PAGE_SIZE=128K
14+
- QDB_CAIRO_O3_COLUMN_MEMORY_SIZE=128K
15+
- QDB_CAIRO_WRITER_DATA_INDEX_KEY_APPEND_PAGE_SIZE=128K
16+
- QDB_CAIRO_WRITER_DATA_INDEX_VALUE_APPEND_PAGE_SIZE=128K
17+
18+
envoy:
19+
image: envoyproxy/envoy-contrib:v1.32-latest
20+
container_name: envoy
21+
volumes:
22+
- ./envoy.yaml:/etc/envoy/envoy.yaml:ro
23+
- ./certs:/etc/envoy/certs:ro
24+
ports:
25+
- "9000:9000"
26+
- "8812:8812"
27+
- "9901:9901"
28+
networks:
29+
- my_private
30+
depends_on:
31+
- questdb
32+
user: "101:101"
33+
security_opt:
34+
- no-new-privileges:true
35+
36+
37+
networks:
38+
my_private:
39+
driver: bridge
40+

‎questdb/envoy.yaml

+97
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
static_resources:
2+
listeners:
3+
- name: https_listener
4+
address:
5+
socket_address:
6+
address: 0.0.0.0
7+
port_value: 9000
8+
filter_chains:
9+
- filters:
10+
- name: envoy.filters.network.http_connection_manager
11+
typed_config:
12+
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
13+
stat_prefix: questdb_https
14+
codec_type: AUTO
15+
route_config:
16+
name: questdb_routes
17+
virtual_hosts:
18+
- name: questdb_service
19+
domains: ["*"]
20+
routes:
21+
- match:
22+
prefix: "/"
23+
route:
24+
cluster: questdb_cluster
25+
timeout: 0s
26+
http_filters:
27+
- name: envoy.filters.http.router
28+
typed_config:
29+
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
30+
transport_socket:
31+
name: envoy.transport_sockets.tls
32+
typed_config:
33+
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
34+
common_tls_context:
35+
tls_certificates:
36+
- certificate_chain:
37+
filename: "/etc/envoy/certs/cert.pem"
38+
private_key:
39+
filename: "/etc/envoy/certs/key.pem"
40+
41+
- name: postgres_listener
42+
address:
43+
socket_address:
44+
address: 0.0.0.0
45+
port_value: 8812
46+
filter_chains:
47+
- filters:
48+
- name: envoy.filters.network.postgres_proxy
49+
typed_config:
50+
"@type": type.googleapis.com/envoy.extensions.filters.network.postgres_proxy.v3alpha.PostgresProxy
51+
stat_prefix: postgres_stats
52+
terminate_ssl: true
53+
- name: envoy.filters.network.tcp_proxy
54+
typed_config:
55+
"@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
56+
stat_prefix: postgres_tcp
57+
cluster: questdb_postgres
58+
transport_socket:
59+
name: envoy.transport_sockets.starttls
60+
typed_config:
61+
"@type": type.googleapis.com/envoy.extensions.transport_sockets.starttls.v3.StartTlsConfig
62+
tls_socket_config:
63+
common_tls_context:
64+
tls_certificates:
65+
- certificate_chain:
66+
filename: "/etc/envoy/certs/cert.pem"
67+
private_key:
68+
filename: "/etc/envoy/certs/key.pem"
69+
70+
clusters:
71+
- name: questdb_cluster
72+
connect_timeout: 30s
73+
type: STRICT_DNS
74+
lb_policy: ROUND_ROBIN
75+
load_assignment:
76+
cluster_name: questdb_cluster
77+
endpoints:
78+
- lb_endpoints:
79+
- endpoint:
80+
address:
81+
socket_address:
82+
address: questdb
83+
port_value: 9000
84+
85+
- name: questdb_postgres
86+
connect_timeout: 30s
87+
type: STRICT_DNS
88+
lb_policy: ROUND_ROBIN
89+
load_assignment:
90+
cluster_name: questdb_postgres
91+
endpoints:
92+
- lb_endpoints:
93+
- endpoint:
94+
address:
95+
socket_address:
96+
address: questdb
97+
port_value: 8812

‎setup.sh

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
3+
# Docker installation
4+
sudo apt-get update
5+
sudo apt-get install -y ca-certificates curl
6+
sudo install -m 0755 -d /etc/apt/keyrings
7+
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
8+
sudo chmod a+r /etc/apt/keyrings/docker.asc
9+
10+
11+
echo \
12+
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
13+
$(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
14+
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
15+
sudo apt-get update
16+
17+
sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
18+
19+
20+
if ! getent group docker > /dev/null 2>&1; then
21+
sudo groupadd docker
22+
fi
23+
24+
25+
if ! groups $USER | grep -q docker; then
26+
sudo usermod -aG docker $USER
27+
echo "User added to docker group. Please log out and log back in for changes to take effect."
28+
fi
29+
30+
31+
32+
# Add Ubuntu repository
33+
sudo sh -c 'echo "deb http://archive.ubuntu.com/ubuntu jammy main universe" > /etc/apt/sources.list'
34+
35+
# Update and install ZFS utils
36+
sudo apt-get update
37+
sudo apt-get install -y zfsutils-linux
38+
39+
# Check if /dev/sdb exists
40+
if [ ! -e /dev/sdb ]; then
41+
echo "Error: /dev/sdb does not exist"
42+
exit 1
43+
fi
44+
45+
# Create GPT label
46+
echo "mklabel gpt" | sudo parted /dev/sdb
47+
48+
# Create ZFS pool and dataset
49+
sudo zpool create -o ashift=12 -O atime=off -O canmount=off -O compression=lz4 data_pool /dev/sdb
50+
if [ $? -eq 0 ]; then
51+
sudo zfs create -o mountpoint=/questdb_zfs data_pool/primary
52+
else
53+
echo "Error: Failed to create ZFS pool"
54+
exit 1
55+
fi

‎terraform/main.tf

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
terraform {
2+
required_providers {
3+
google = {
4+
source = "hashicorp/google"
5+
version = "6.8.0"
6+
}
7+
}
8+
}
9+
10+
provider "google" {
11+
#credentials = file(var.credentials_file)
12+
project = var.project
13+
region = var.region
14+
}
15+
16+
17+
resource "google_compute_network" "qdb-vpc" {
18+
name = "qdb-vpc"
19+
routing_mode = "REGIONAL"
20+
auto_create_subnetworks = false
21+
network_firewall_policy_enforcement_order = "AFTER_CLASSIC_FIREWALL"
22+
}
23+
24+
resource "google_compute_subnetwork" "qdb-subnet-with-logging" {
25+
name = "qdb-subnet-with-logging"
26+
ip_cidr_range = "10.2.0.0/16"
27+
region = var.region
28+
network = google_compute_network.qdb-vpc.id
29+
stack_type = "IPV4_ONLY"
30+
31+
log_config {
32+
aggregation_interval = "INTERVAL_10_MIN"
33+
flow_sampling = 0.5
34+
metadata = "INCLUDE_ALL_METADATA"
35+
}
36+
}
37+
38+
resource "google_compute_route" "qdb-public-route" {
39+
name = "qdb-public-route"
40+
network = google_compute_network.qdb-vpc.id
41+
dest_range = "0.0.0.0/0" # Destination IP range
42+
priority = 1000
43+
next_hop_gateway = "default-internet-gateway" # Specify the next hop
44+
}
45+
46+
47+
resource "google_compute_firewall" "qdb-firewall" {
48+
name = "qdb-firewall"
49+
description = "Creates firewall rule targeting tagged instances"
50+
network = google_compute_network.qdb-vpc.id
51+
52+
allow {
53+
protocol = "all"
54+
}
55+
56+
source_ranges = [var.my_ip]
57+
58+
target_tags = ["qdb"]
59+
}
60+
61+
62+
resource "google_compute_instance" "qdb-ubuntu" {
63+
name = "qdb-ubuntu"
64+
machine_type = "n2d-standard-2"
65+
zone = "australia-southeast1-a"
66+
67+
tags = ["qdb"]
68+
69+
boot_disk {
70+
initialize_params {
71+
image = "ubuntu-2410-oracular-amd64-v20241021"
72+
size = 10
73+
type = "pd-balanced"
74+
}
75+
auto_delete = true
76+
}
77+
78+
attached_disk {
79+
source = google_compute_disk.persistent-data.id
80+
device_name = google_compute_disk.persistent-data.name
81+
}
82+
83+
84+
network_interface {
85+
network = google_compute_network.qdb-vpc.id
86+
subnetwork = google_compute_subnetwork.qdb-subnet-with-logging.id
87+
access_config {
88+
network_tier = "STANDARD"
89+
}
90+
91+
}
92+
}
93+
94+
output "public-ipv4" {
95+
value = google_compute_instance.qdb-ubuntu.network_interface.0.access_config.0.nat_ip
96+
}
97+
98+
99+
resource "google_compute_disk" "persistent-data" {
100+
name = "persistent-data"
101+
type = "pd-balanced"
102+
zone = "australia-southeast1-a"
103+
size = "20"
104+
}
105+

‎terraform/terraform.tfvars

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
project = "your-project-id"
2+
credentials_file = "<FILE>"
3+
my_ip = "0.0.0.0/0"

‎terraform/variables.tf

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
variable "project" {}
2+
3+
variable "credentials_file" {}
4+
5+
variable "my_ip" {}
6+
7+
8+
variable "region" {
9+
default = "australia-southeast1"
10+
}
11+
12+
variable "zone" {
13+
default = "australia-southeast1"
14+
}
15+

0 commit comments

Comments
 (0)