Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

start VTGate and VTTablet with config file #611

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions config/wescale/bak/vtgate.cnf
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

bak dir is unnecessary

Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
[vtgate]
v=100
tablet_types_to_wait=PRIMARY
gateway_initial_tablet_timeout=3s
healthcheck_timeout=2s
srv_topo_timeout=1s
grpc_keepalive_time=10s
grpc_keepalive_timeout=10s
tablet_refresh_interval=1m
read_write_splitting_policy=disable
read_write_splitting_ratio=100
read_after_write_consistency=SESSION
read_after_write_timeout=30.0
enable_buffer=true
buffer_size=10000
buffer_window=180s
buffer_max_failover_duration=360s
buffer_min_time_between_failovers=0s
mysql_auth_server_impl=none
mysql_auth_server_static_file=
mysql_server_require_secure_transport=false
ddl_strategy=direct
enable_display_sql_execution_vttablets=false
enable_read_write_split_for_read_only_txn=false
enable_interception_for_dml_without_where=true
mysql_server_ssl_ca=
mysql_server_ssl_cert=
mysql_server_ssl_key=

enable_interception_for_dml_without_where=false

# AutoScale & AutoSuspend Params
enable_auto_suspend=false
auto_suspend_timeout=5m

enable_auto_scale=false
auto_scale_decision_making_interval=5s
auto_scale_compute_unit_lower_bound=0.5
auto_scale_compute_unit_upper_bound=10
auto_scale_cpu_ratio=0.9
auto_scale_memory_ratio=0.75
auto_scale_use_relaxed_cpu_memory_ratio=false

auto_scale_cluster_namespace=default
auto_scale_data_node_pod_name=mycluster-wesql-0-0
auto_scale_data_node_stateful_set_name=mycluster-wesql-0
auto_scale_logger_node_pod_name=mycluster-wesql-1-0,mycluster-wesql-2-0
auto_scale_logger_node_stateful_set_name=mycluster-wesql-1,mycluster-wesql-2
28 changes: 28 additions & 0 deletions config/wescale/bak/vttablet.cnf
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
[vttablet]
health_check_interval=1s
shard_sync_retry_delay=1s
remote_operation_timeout=1s
db_connect_timeout_ms=500
table_acl_config_mode=simple
enable_logs=true
enable_query_log=true
table_acl_config=
queryserver_config_strict_table_acl=false
table_acl_config_reload_interval=5s
enforce_tableacl_config=false

queryserver_config_transaction_cap=20
queryserver_config_pool_size=16
queryserver_config_stream_pool_size=4
dba_pool_size=5
queryserver_pool_autoscale_enable=true
queryserver_pool_autoscale_tx_pool_percentage=70

non_transactional_dml_database_pool_size=3
non_transactional_dml_default_batch_size=2000
non_transactional_dml_default_batch_interval=1
non_transactional_dml_table_gc_interval=24
non_transactional_dml_job_manager_running_interval=24
non_transactional_dml_throttle_check_interval=250
non_transactional_dml_batch_size_threshold=10000
non_transactional_dml_batch_size_threshold_ratio=0.5
16 changes: 15 additions & 1 deletion config/wescale/default/vtgate.cnf
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,18 @@ auto_scale_cluster_namespace=default
auto_scale_data_node_pod_name=mycluster-wesql-0-0
auto_scale_data_node_stateful_set_name=mycluster-wesql-0
auto_scale_logger_node_pod_name=mycluster-wesql-1-0,mycluster-wesql-2-0
auto_scale_logger_node_stateful_set_name=mycluster-wesql-1,mycluster-wesql-2
auto_scale_logger_node_stateful_set_name=mycluster-wesql-1,mycluster-wesql-2

topo_implementation=etcd2
topo_global_server_address=127.0.0.1:2379
topo_global_root=/vitess/global
alsologtostderr=true
log_queries_to_file=vtdataroot/vtgate/vtgate_querylog.txt
port=15001
grpc_port=15991
mysql_server_port=15306
mysql_server_socket_path=/tmp/mysql.sock
cell=zone1
cells_to_watch=zone1
service_map=grpc-vtgateservice
pid_file=vtdataroot/vtgate/vtgate.pid
31 changes: 30 additions & 1 deletion config/wescale/default/vttablet.cnf
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,33 @@ non_transactional_dml_table_gc_interval=24
non_transactional_dml_job_manager_running_interval=24
non_transactional_dml_throttle_check_interval=250
non_transactional_dml_batch_size_threshold=10000
non_transactional_dml_batch_size_threshold_ratio=0.5
non_transactional_dml_batch_size_threshold_ratio=0.5

topo_implementation=etcd2
topo_global_server_address=127.0.0.1:2379
topo_global_root=/vitess/global
alsologtostderr=true
log_dir=/vttablet
log_queries_to_file=vtdataroot/vttablet/vttablet_0000000000_querylog.txt
tablet-path=zone1-0000000000
tablet_hostname=127.0.0.1
init_tablet_type=replica
enable_replication_reporter=true
backup_storage_implementation=file
file_backup_storage_root=vtdataroot/vttablet/backups
port=15100
db_port=3306
db_host=127.0.0.1
db_allprivs_user=root
db_allprivs_password=passwd
db_dba_user=root
db_dba_password=passwd
db_app_user=root
db_app_password=passwd
db_filtered_user=root
db_filtered_password=passwd
grpc_port=16100
service_map=grpc-queryservice,grpc-tabletmanager,grpc-updatestream
pid_file=vtdataroot/vttablet/vttablet.pid
vtctld_addr=http://127.0.0.1:15000/
disable_active_reparents=true
75 changes: 60 additions & 15 deletions examples/wesql-server/vtgate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,20 +12,65 @@ config_path=${CONFIG_PATH}
topology_flags=${TOPOLOGY_FLAGS:-'--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global'}

VTDATAROOT=$VTDATAROOT/vtgate
mkdir -p $VTDATAROOT
mkdir -p "$VTDATAROOT"

original_config="$config_path/../bak/vtgate.cnf"
temp_config="$config_path/vtgate.cnf"

# 检查原始配置文件是否存在
if [ ! -f "$original_config" ]; then
echo "配置文件 $original_config 不存在。请确保 CONFIG_PATH 设置正确并且配置文件存在。"
exit 1
fi

# 拷贝原始配置文件到临时文件
cp "$original_config" "$temp_config"

printf "\n\n" >> "$temp_config"

# 解析 topology_flags 并转换为 key=value 形式追加到临时配置文件
# 使用数组来正确处理带有空格的值
IFS=' ' read -r -a flags_array <<< "$topology_flags"

i=0
while [ $i -lt ${#flags_array[@]} ]; do
flag="${flags_array[$i]}"
if [[ "$flag" == --* ]]; then
key="${flag#--}"
# 检查是否为 --key=value 形式
if [[ "$key" == *"="* ]]; then
key="${key%%=*}"
value="${flag#*=}"
else
# 检查下一个参数是否存在且不是另一个 flag
next_index=$((i + 1))
if [ $next_index -lt ${#flags_array[@]} ] && [[ "${flags_array[$next_index]}" != --* ]]; then
value="${flags_array[$next_index]}"
i=$next_index
else
# 布尔值标志
value=true
fi
fi
echo "$key=$value" >> "$temp_config"
fi
i=$((i + 1))
done

# 追加其他标志到临时配置文件
cat <<EOL >> "$temp_config"
alsologtostderr=true
log_queries_to_file=$VTDATAROOT/vtgate_querylog.txt
port=$web_port
grpc_port=$grpc_port
mysql_server_port=$mysql_server_port
mysql_server_socket_path=$mysql_server_socket_path
cell=$cell
cells_to_watch=$cell
service_map=grpc-vtgateservice
pid_file=$VTDATAROOT/vtgate.pid
EOL

# 启动 vtgate,仅使用 --config_path 参数
vtgate \
$topology_flags \
--v 100 \
--alsologtostderr \
--log_dir $VTDATAROOT \
--log_queries_to_file $VTDATAROOT/vtgate_querylog.txt \
--port $web_port \
--grpc_port $grpc_port \
--mysql_server_port $mysql_server_port \
--mysql_server_socket_path $mysql_server_socket_path \
--cell $cell \
--cells_to_watch $cell \
--service_map 'grpc-vtgateservice' \
--pid_file $VTDATAROOT/vtgate.pid \
--config_path $config_path
--config_path "$config_path"
31 changes: 31 additions & 0 deletions examples/wesql-server/vtgate_using_flags.sh
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Avoid introducing "vtgate_using_flags.sh" in wesql-server if it's not being actively utilized.

Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash

echo "Starting vtgate..."

cell=${CELL:-'zone1'}
web_port=${VTGATE_WEB_PORT:-'15001'}
grpc_port=${VTGATE_GRPC_PORT:-'15991'}
mysql_server_port=${VTGATE_MYSQL_PORT:-'15306'}
mysql_server_socket_path="/tmp/mysql.sock"
config_path=${CONFIG_PATH}

topology_flags=${TOPOLOGY_FLAGS:-'--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global'}

VTDATAROOT=$VTDATAROOT/vtgate
mkdir -p $VTDATAROOT

vtgate \
$topology_flags \
--v 100 \
--alsologtostderr \
--log_dir $VTDATAROOT \
--log_queries_to_file $VTDATAROOT/vtgate_querylog.txt \
--port $web_port \
--grpc_port $grpc_port \
--mysql_server_port $mysql_server_port \
--mysql_server_socket_path $mysql_server_socket_path \
--cell $cell \
--cells_to_watch $cell \
--service_map 'grpc-vtgateservice' \
--pid_file $VTDATAROOT/vtgate.pid \
--config_path $config_path
110 changes: 78 additions & 32 deletions examples/wesql-server/vttablet.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,41 +15,87 @@ vtctld_web_port=${VTCTLD_WEB_PORT:-'15000'}
tablet_hostname='127.0.0.1'
config_path=${CONFIG_PATH}

printf -v alias '%s-%010d' $cell $uid
printf -v tablet_dir 'vt_%010d' $uid
printf -v tablet_logfile 'vttablet_%010d_querylog.txt' $uid
printf -v alias '%s-%010d' "$cell" "$uid"
printf -v tablet_dir 'vt_%010d' "$uid"
printf -v tablet_logfile 'vttablet_%010d_querylog.txt' "$uid"

topology_flags=${TOPOLOGY_FLAGS:-'--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global'}

VTDATAROOT=$VTDATAROOT/vttablet
mkdir -p $VTDATAROOT
mkdir -p "$VTDATAROOT"

# 定义配置文件路径
original_config="$config_path/../bak/vttablet.cnf"
temp_config="$config_path/vttablet.cnf"

# 检查原始配置文件是否存在
if [ ! -f "$original_config" ]; then
echo "配置文件 $original_config 不存在。请确保 CONFIG_PATH 设置正确并且配置文件存在。"
exit 1
fi

# 拷贝原始配置文件到临时文件
cp "$original_config" "$temp_config"

printf "\n\n" >> "$temp_config"

# 解析 topology_flags 并转换为 key=value 形式追加到临时配置文件
# 使用数组来正确处理带有空格的值
IFS=' ' read -r -a flags_array <<< "$topology_flags"

i=0
while [ $i -lt ${#flags_array[@]} ]; do
flag="${flags_array[$i]}"
if [[ "$flag" == --* ]]; then
key="${flag#--}"
# 检查是否为 --key=value 形式
if [[ "$key" == *"="* ]]; then
key="${key%%=*}"
value="${flag#*=}"
else
# 检查下一个参数是否存在且不是另一个 flag
next_index=$((i + 1))
if [ $next_index -lt ${#flags_array[@]} ] && [[ "${flags_array[$next_index]}" != --* ]]; then
value="${flags_array[$next_index]}"
i=$next_index
else
# 布尔值标志
value=true
fi
fi
echo "$key=$value" >> "$temp_config"
fi
i=$((i + 1))
done

# 追加其他标志到临时配置文件
cat <<EOL >> "$temp_config"
alsologtostderr=true
log_queries_to_file=$VTDATAROOT/$tablet_logfile
tablet-path=$alias
tablet_hostname=$tablet_hostname
init_tablet_type=replica
enable_replication_reporter=true
backup_storage_implementation=file
file_backup_storage_root=$VTDATAROOT/backups
port=$port
db_port=$mysql_port
db_host=$mysql_host
db_allprivs_user=$mysql_root
db_allprivs_password=$mysql_root_passwd
db_dba_user=$mysql_root
db_dba_password=$mysql_root_passwd
db_app_user=$mysql_root
db_app_password=$mysql_root_passwd
db_filtered_user=$mysql_root
db_filtered_password=$mysql_root_passwd
grpc_port=$grpc_port
service_map=grpc-queryservice,grpc-tabletmanager,grpc-updatestream
pid_file=$VTDATAROOT/vttablet.pid
vtctld_addr=http://$vtctld_host:$vtctld_web_port/
disable_active_reparents=true
EOL

# 启动 vttablet,仅使用 --config_path 参数
vttablet \
$topology_flags \
--v 100 \
--alsologtostderr \
--log_dir $VTDATAROOT \
--log_queries_to_file $VTDATAROOT/$tablet_logfile \
--tablet-path $alias \
--tablet_hostname "$tablet_hostname" \
--init_tablet_type replica \
--enable_replication_reporter \
--backup_storage_implementation file \
--file_backup_storage_root $VTDATAROOT/backups \
--port $port \
--db_port $mysql_port \
--db_host $mysql_host \
--db_allprivs_user $mysql_root \
--db_allprivs_password $mysql_root_passwd \
--db_dba_user $mysql_root \
--db_dba_password $mysql_root_passwd \
--db_app_user $mysql_root \
--db_app_password $mysql_root_passwd \
--db_filtered_user $mysql_root \
--db_filtered_password $mysql_root_passwd \
--grpc_port $grpc_port \
--service_map 'grpc-queryservice,grpc-tabletmanager,grpc-updatestream' \
--pid_file $VTDATAROOT/vttablet.pid \
--vtctld_addr http://$vtctld_host:$vtctld_web_port/ \
--disable_active_reparents \
--config_path $config_path
--config_path "$config_path"
Loading
Loading