diff --git a/config.example.yaml b/config.example.yaml index ed696648..8a539830 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -106,8 +106,8 @@ clickhouse: username: "default" password: "123456" database: "openflare" - max_idle_conn: 10 - max_open_conn: 100 + max_idle_conn: 20 + max_open_conn: 50 conn_max_lifetime: 3600 dial_timeout: 5 - block_buffer_size: 10 + block_buffer_size: 100 diff --git a/internal/config/config.go b/internal/config/config.go index 268c7997..df5ed8cf 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -135,10 +135,10 @@ func applyClickHouseDefaults(c *configModel) { c.ClickHouse.Username = "default" } if c.ClickHouse.MaxIdleConn <= 0 { - c.ClickHouse.MaxIdleConn = 10 + c.ClickHouse.MaxIdleConn = 20 } if c.ClickHouse.MaxOpenConn <= 0 { - c.ClickHouse.MaxOpenConn = 100 + c.ClickHouse.MaxOpenConn = 50 } if c.ClickHouse.ConnMaxLifetime <= 0 { c.ClickHouse.ConnMaxLifetime = 3600 @@ -147,7 +147,7 @@ func applyClickHouseDefaults(c *configModel) { c.ClickHouse.DialTimeout = 5 } if c.ClickHouse.BlockBufferSize == 0 { - c.ClickHouse.BlockBufferSize = 10 + c.ClickHouse.BlockBufferSize = 100 } } diff --git a/internal/db/clickhouse.go b/internal/db/clickhouse.go index bec6c52c..092203f1 100644 --- a/internal/db/clickhouse.go +++ b/internal/db/clickhouse.go @@ -24,8 +24,10 @@ import ( ) const ( - clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒) - clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数 + clickhouseMaxExecTime = 60 // ClickHouse 最大执行时间(秒) + clickhouseReadTimeoutFactor = 2 // ReadTimeout 为 DialTimeout 的倍数 + clickhouseAsyncInsertMaxDataSize = 10_000_000 + clickhouseAsyncInsertBusyTimeoutMs = 1000 ) var ( @@ -101,7 +103,11 @@ func buildClickHouseOptions() *clickhouse.Options { Password: cfg.Password, }, Settings: clickhouse.Settings{ - "max_execution_time": clickhouseMaxExecTime, + "max_execution_time": clickhouseMaxExecTime, + "async_insert": 1, + "wait_for_async_insert": 0, + "async_insert_max_data_size": clickhouseAsyncInsertMaxDataSize, + "async_insert_busy_timeout_ms": clickhouseAsyncInsertBusyTimeoutMs, }, Compression: &clickhouse.Compression{ Method: clickhouse.CompressionLZ4, @@ -131,6 +137,12 @@ func buildClickHouseDSN() string { query.Set("dial_timeout", fmt.Sprintf("%ds", cfg.DialTimeout)) query.Set("read_timeout", fmt.Sprintf("%ds", cfg.DialTimeout*clickhouseReadTimeoutFactor)) query.Set("max_execution_time", strconv.Itoa(clickhouseMaxExecTime)) + // GORM uses clickhouse-go ParseDSN; unknown params are passed as server Settings. + query.Set("async_insert", "1") + query.Set("wait_for_async_insert", "0") + query.Set("async_insert_max_data_size", strconv.Itoa(clickhouseAsyncInsertMaxDataSize)) + query.Set("async_insert_busy_timeout_ms", strconv.Itoa(clickhouseAsyncInsertBusyTimeoutMs)) + query.Set("max_insert_block_size", strconv.Itoa(clickhouseAsyncInsertMaxDataSize)) chURL.RawQuery = query.Encode() return chURL.String()