Skip to main content

14 posts tagged with "clickhouse"

View All Tags

clickhouse 物化视图

· 12 min read
Thread 3 "HTTPHandler" hit Breakpoint 2, DB::buildPushingToViewsChain (storage=..., metadata_snapshot=..., context=..., query_ptr=..., no_destination=false, thread_status_holder=..., running_group=..., elapsed_counter_ms=0x0, async_insert=false, live_view_header=...) at /ssd/ClickHouse/src/Processors/Transforms/buildPushingToViewsChain.cpp:307
307                 if (lock == nullptr)
(gdb) bt
#0  DB::buildPushingToViewsChain (storage=..., metadata_snapshot=..., context=..., query_ptr=..., no_destination=false, thread_status_holder=..., running_group=..., elapsed_counter_ms=0x0, async_insert=false, live_view_header=...)
    at /ssd/ClickHouse/src/Processors/Transforms/buildPushingToViewsChain.cpp:307
#1  0x000000001cc8a4e0 in DB::InterpreterInsertQuery::buildSink (this=0x7ffe5a9e63f0, table=..., metadata_snapshot=..., thread_status_holder=..., running_group=..., elapsed_counter_ms=0x0) at /ssd/ClickHouse/src/Interpreters/InterpreterInsertQuery.cpp:311
#2  0x000000001cc8cbdb in DB::InterpreterInsertQuery::execute (this=0x7ffe5a9e63f0) at /ssd/ClickHouse/src/Interpreters/InterpreterInsertQuery.cpp:554
#3  0x000000001d2d9d4a in DB::executeQueryImpl (begin=0x7ffe5aa40c00 "insert into push.site_sku_user_log_distributed (site_sku , site ,sku_id ,user_id  , is_in_cart ) values \r\n( 'us_1' , 'us' , '1' , '2'  , '1'  )", '\245' <repeats 57 times>...,
    end=0x7ffe5aa40c8f '\245' <repeats 113 times>, 'Z' <repeats 87 times>..., context=..., internal=false, stage=DB::QueryProcessingStage::Complete, istr=0x7ffe5a8fc020) at /ssd/ClickHouse/src/Interpreters/executeQuery.cpp:1096
#4  0x000000001d2dbdd1 in DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (DB::QueryResultDetails const&)>, std::__1::optional<DB::FormatSettings> const&, std::__1::function<void (DB::IOutputFormat&)>) (istr=..., ostr=..., allow_into_outfile=false, context=..., set_result_details=..., output_format_settings=..., handle_exception_in_output_format=...) at /ssd/ClickHouse/src/Interpreters/executeQuery.cpp:1351
#5  0x000000001e7b8f91 in DB::HTTPHandler::processQuery (this=0x7ffe43e3ce80, request=..., params=..., response=..., used_output=..., query_scope=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:884
#6  0x000000001e7bbb67 in DB::HTTPHandler::handleRequest (this=0x7ffe43e3ce80, request=..., response=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:1078
#7  0x000000001e8644dc in DB::HTTPServerConnection::run (this=0x7ffe43e3cd40) at /ssd/ClickHouse/src/Server/HTTP/HTTPServerConnection.cpp:68
#8  0x000000002396f7d9 in Poco::Net::TCPServerConnection::start (this=0x7ffe43e3cd40) at /ssd/ClickHouse/base/poco/Net/src/TCPServerConnection.cpp:43
#9  0x000000002397001c in Poco::Net::TCPServerDispatcher::run (this=0x7ffe5a8c0f00) at /ssd/ClickHouse/base/poco/Net/src/TCPServerDispatcher.cpp:115
#10 0x0000000023b5ac14 in Poco::PooledThread::run (this=0x7ffff71cab80) at /ssd/ClickHouse/base/poco/Foundation/src/ThreadPool.cpp:188
#11 0x0000000023b579ba in Poco::(anonymous namespace)::RunnableHolder::run (this=0x7ffff70019c0) at /ssd/ClickHouse/base/poco/Foundation/src/Thread.cpp:45
#12 0x0000000023b566be in Poco::ThreadImpl::runnableEntry (pThread=0x7ffff71cabb8) at /ssd/ClickHouse/base/poco/Foundation/src/Thread_POSIX.cpp:335
#13 0x00007ffff7c94ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#14 0x00007ffff7d26a40 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

clickhouse 如何写入​

### 删除表
drop table  TT sync
### 建表语句
CREATE TABLE TT
(
    `A` Int64,
    `B` FixedString(10)
)

ENGINE = MergeTree
order by A
SETTINGS index_granularity = 2

#### 查询语句

select * from  TT where A = 2

### 写入语句

insert into   TT  (A,B) values( 4 , 'dab2a')

clickhouse 写入

(gdb) bt
#0  DB::SerializationFixedString::serializeBinaryBulk (this=0x7ffff6388f58, column=..., ostr=..., offset=0, limit=1) at /ssd/ClickHouse/src/DataTypes/Serializations/SerializationFixedString.cpp:75
#1  0x000000001bb38562 in DB::ISerialization::serializeBinaryBulkWithMultipleStreams (this=0x7ffff6388f58, column=..., offset=0, limit=1, settings=...) at /ssd/ClickHouse/src/DataTypes/Serializations/ISerialization.cpp:114
#2  0x000000001e1632d4 in DB::(anonymous namespace)::writeColumnSingleGranule(DB::ColumnWithTypeAndName const&, std::__1::shared_ptr<DB::ISerialization const> const&, std::__1::function<DB::WriteBuffer* (DB::ISerialization::SubstreamPath const&)>, unsigned long, unsigned long) (column=..., serialization=..., stream_getter=..., from_row=0, number_of_rows=1) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:156
#3  0x000000001e1630f9 in DB::MergeTreeDataPartWriterCompact::writeDataBlock (this=0x7ffe42558800, block=..., granules=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:244
#4  0x000000001e162caf in DB::MergeTreeDataPartWriterCompact::writeDataBlockPrimaryIndexAndSkipIndices (this=0x7ffe42558800, block=..., granules_to_write=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:194
#5  0x000000001e16351e in DB::MergeTreeDataPartWriterCompact::fillDataChecksums (this=0x7ffe42558800, checksums=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:268
#6  0x000000001e1641ba in DB::MergeTreeDataPartWriterCompact::fillChecksums (this=0x7ffe42558800, checksums=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:430
#7  0x000000001e3ccfda in DB::MergedBlockOutputStream::finalizePartAsync (this=0x7ffed02cdfe0, new_part=..., sync=false, total_columns_list=0x0, additional_column_checksums=0x0) at /ssd/ClickHouse/src/Storages/MergeTree/MergedBlockOutputStream.cpp:151
#8  0x000000001e412cf6 in DB::MergeTreeDataWriter::writeTempPartImpl (this=0x7ffee55279a0, block_with_partition=..., metadata_snapshot=..., context=..., block_number=1, need_tmp_prefix=true) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:592
#9  0x000000001e40ffdd in DB::MergeTreeDataWriter::writeTempPart (this=0x7ffee55279a0, block=..., metadata_snapshot=..., context=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:385
#10 0x000000001e587b8c in DB::MergeTreeSink::consume (this=0x7ffe3d9dc098, chunk=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeSink.cpp:87
#11 0x000000001ef78c5e in DB::SinkToStorage::onConsume (this=0x7ffe3d9dc098, chunk=...) at /ssd/ClickHouse/src/Processors/Sinks/SinkToStorage.cpp:24
#12 0x000000001ee0a83f in DB::ExceptionKeepingTransform::work()::$_1::operator()() const (this=0x7ffed85e97c8) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#13 0x000000001ee0a7f5 in std::__1::__invoke[abi:v15000]<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__f=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:394
#14 0x000000001ee0a7d5 in std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__args=...)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:479
#15 0x000000001ee0a7b5 in std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>::operator()[abi:v15000]() (this=0x7ffed85e97c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:235
#16 0x000000001ee0a77d in std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()> >(std::__1::__function::__policy_storage const*) (__buf=0x7ffed85e97c8)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:716
#17 0x0000000013ac9556 in std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const (this=0x7ffed85e97c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:848
#18 0x0000000013ac8a35 in std::__1::function<void ()>::operator()() const (this=0x7ffed85e97c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:1187
#19 0x000000001ee0a076 in DB::runStep(std::__1::function<void ()>, DB::ThreadStatus*, std::__1::atomic<unsigned long>*) (step=..., thread_status=0x0, elapsed_ms=0x0) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:114
#20 0x000000001ee09abe in DB::ExceptionKeepingTransform::work (this=0x7ffe3d9dc098) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#21 0x000000001e8f2ca3 in DB::executeJob (node=0x7ffed02d4800, read_progress_callback=0x7ffed0265140) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:47
#22 0x000000001e8f29e0 in DB::ExecutionThreadContext::executeTask (this=0x7ffed025b600) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:95
#23 0x000000001e8d7761 in DB::PipelineExecutor::executeStepImpl (this=0x7ffed85ea378, thread_num=0, yield_flag=0x0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:273
#24 0x000000001e8d7a77 in DB::PipelineExecutor::executeSingleThread (this=0x7ffed85ea378, thread_num=0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:239
#25 0x000000001e8d6771 in DB::PipelineExecutor::executeImpl (this=0x7ffed85ea378, num_threads=1, concurrency_control=true) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:399
#26 0x000000001e8d5d76 in DB::PipelineExecutor::execute (this=0x7ffed85ea378, num_threads=1, concurrency_control=true) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:111
#27 0x000000001e8d3daf in DB::CompletedPipelineExecutor::execute (this=0x7ffed85ea7e0) at /ssd/ClickHouse/src/Processors/Executors/CompletedPipelineExecutor.cpp:110
#28 0x000000001d2dcaa0 in DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (DB::QueryResultDetails const&)>, std::__1::optional<DB::FormatSettings> const&, std::__1::function<void (DB::IOutputFormat&)>) (istr=..., ostr=..., allow_into_outfile=false, context=..., set_result_details=..., output_format_settings=..., handle_exception_in_output_format=...) at /ssd/ClickHouse/src/Interpreters/executeQuery.cpp:1448
#29 0x000000001e7b8f91 in DB::HTTPHandler::processQuery (this=0x7ffe5e146d20, request=..., params=..., response=..., used_output=..., query_scope=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:884
#30 0x000000001e7bbb67 in DB::HTTPHandler::handleRequest (this=0x7ffe5e146d20, request=..., response=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:1078
#31 0x000000001e8644dc in DB::HTTPServerConnection::run (this=0x7ffe5e146be0) at /ssd/ClickHouse/src/Server/HTTP/HTTPServerConnection.cpp:68
#32 0x000000002396f7d9 in Poco::Net::TCPServerConnection::start (this=0x7ffe5e146be0) at /ssd/ClickHouse/base/poco/Net/src/TCPServerConnection.cpp:43
#33 0x000000002397001c in Poco::Net::TCPServerDispatcher::run (this=0x7ffe5e1a4f00) at /ssd/ClickHouse/base/poco/Net/src/TCPServerDispatcher.cpp:115
#34 0x0000000023b5ac14 in Poco::PooledThread::run (this=0x7ffff71cb080) at /ssd/ClickHouse/base/poco/Foundation/src/ThreadPool.cpp:188
#35 0x0000000023b579ba in Poco::(anonymous namespace)::RunnableHolder::run (this=0x7ffff70019e0) at /ssd/ClickHouse/base/poco/Foundation/src/Thread.cpp:45
#36 0x0000000023b566be in Poco::ThreadImpl::runnableEntry (pThread=0x7ffff71cb0b8) at /ssd/ClickHouse/base/poco/Foundation/src/Thread_POSIX.cpp:335
#37 0x00007ffff7c94ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#38 0x00007ffff7d26a40 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

打开bin文件

(gdb) bt
#0  DB::WriteBufferFromFile::WriteBufferFromFile (this=0x7ffece3419a0, file_name_=..., buf_size=1048576, flags=-1, throttler_=..., mode=438, existing_memory=0x0, alignment=0) at /ssd/ClickHouse/src/IO/WriteBufferFromFile.cpp:36
#1  0x000000001bc9c455 in std::__1::make_unique[abi:v15000]<DB::WriteBufferFromFile, std::__1::__fs::filesystem::path, unsigned long&, int&, std::__1::shared_ptr<DB::Throttler> const&>(std::__1::__fs::filesystem::path&&, unsigned long&, int&, std::__1::shared_ptr<DB::Throttler> const&) (__args=..., __args=..., __args=..., __args=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714
#2  0x000000001bc94d7e in DB::DiskLocal::writeFile (this=0x7ffe5baec018, path=..., buf_size=1048576, mode=DB::WriteMode::Rewrite, settings=...) at /ssd/ClickHouse/src/Disks/DiskLocal.cpp:342
#3  0x000000001bcafb03 in DB::FakeDiskTransaction::writeFile (this=0x7ffe5427db88, path=..., buf_size=1048576, mode=DB::WriteMode::Rewrite, settings=...) at /ssd/ClickHouse/src/Disks/FakeDiskTransaction.h:69
#4  0x000000001df32dd9 in DB::DataPartStorageOnDiskFull::writeFile (this=0x7ffed4147918, name=..., buf_size=1048576, mode=DB::WriteMode::Rewrite, settings=...) at /ssd/ClickHouse/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp:132
#5  0x000000001df2fb93 in DB::IDataPartStorage::writeFile (this=0x7ffed4147918, name=..., buf_size=1048576, settings=...) at /ssd/ClickHouse/src/Storages/MergeTree/IDataPartStorage.h:285
#6  0x000000001e1618bd in DB::MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact (this=0x7ffe4e99d400, data_part_=..., columns_list_=..., metadata_snapshot_=..., indices_to_recalc_=..., marks_file_extension_=..., default_codec_=..., settings_=...,
    index_granularity_=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:34
#7  0x000000001e151d10 in std::__1::make_unique[abi:v15000]<DB::MergeTreeDataPartWriterCompact, std::__1::shared_ptr<DB::IMergeTreeDataPart>, DB::NamesAndTypesList&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::shared_ptr<DB::ICompressionCodec> const&, DB::MergeTreeWriterSettings const&, DB::MergeTreeIndexGranularity const&>(std::__1::shared_ptr<DB::IMergeTreeDataPart>&&, DB::NamesAndTypesList&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&, std::__1::shared_ptr<DB::ICompressionCodec> const&, DB::MergeTreeWriterSettings const&, DB::MergeTreeIndexGranularity const&) (__args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714
#8  0x000000001e14efe0 in DB::MergeTreeDataPartCompact::getWriter (this=0x7ffe7f742218, columns_list=..., metadata_snapshot=..., indices_to_recalc=..., default_codec_=..., writer_settings=..., computed_index_granularity=...)
    at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp:74
#9  0x000000001e3cc967 in DB::MergedBlockOutputStream::MergedBlockOutputStream (this=0x7ffe3ee65820, data_part=..., metadata_snapshot_=..., columns_list_=..., skip_indices=..., default_codec_=..., txn=..., reset_columns_=false, blocks_are_granules_size=false,
    write_settings_=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergedBlockOutputStream.cpp:50
#10 0x000000001e416327 in std::__1::make_unique[abi:v15000]<DB::MergedBlockOutputStream, std::__1::shared_ptr<DB::IMergeTreeDataPart>&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, DB::NamesAndTypesList&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > >&, std::__1::shared_ptr<DB::ICompressionCodec>&, std::__1::shared_ptr<DB::MergeTreeTransaction>, bool, bool, DB::WriteSettings>(std::__1::shared_ptr<DB::IMergeTreeDataPart>&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, DB::NamesAndTypesList&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > >&, std::__1::shared_ptr<DB::ICompressionCodec>&, std::__1::shared_ptr<DB::MergeTreeTransaction>&&, bool&&, bool&&, DB::WriteSettings&&) (__args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=...)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714
#11 0x000000001e412851 in DB::MergeTreeDataWriter::writeTempPartImpl (this=0x7ffed1e681a0, block_with_partition=..., metadata_snapshot=..., context=..., block_number=2, need_tmp_prefix=true) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:567
#12 0x000000001e40ffdd in DB::MergeTreeDataWriter::writeTempPart (this=0x7ffed1e681a0, block=..., metadata_snapshot=..., context=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:385
#13 0x000000001e587b8c in DB::MergeTreeSink::consume (this=0x7ffe4e9ccc18, chunk=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeSink.cpp:87
#14 0x000000001ef78c5e in DB::SinkToStorage::onConsume (this=0x7ffe4e9ccc18, chunk=...) at /ssd/ClickHouse/src/Processors/Sinks/SinkToStorage.cpp:24
#15 0x000000001ee0a83f in DB::ExceptionKeepingTransform::work()::$_1::operator()() const (this=0x7ffed8dea7c8) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#16 0x000000001ee0a7f5 in std::__1::__invoke[abi:v15000]<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__f=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:394
#17 0x000000001ee0a7d5 in std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__args=...)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:479
#18 0x000000001ee0a7b5 in std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>::operator()[abi:v15000]() (this=0x7ffed8dea7c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:235
#19 0x000000001ee0a77d in std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()> >(std::__1::__function::__policy_storage const*) (__buf=0x7ffed8dea7c8)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:716
#20 0x0000000013ac9556 in std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const (this=0x7ffed8dea7c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:848
#21 0x0000000013ac8a35 in std::__1::function<void ()>::operator()() const (this=0x7ffed8dea7c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:1187
#22 0x000000001ee0a076 in DB::runStep(std::__1::function<void ()>, DB::ThreadStatus*, std::__1::atomic<unsigned long>*) (step=..., thread_status=0x0, elapsed_ms=0x0) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:114
#23 0x000000001ee09abe in DB::ExceptionKeepingTransform::work (this=0x7ffe4e9ccc18) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#24 0x000000001e8f2ca3 in DB::executeJob (node=0x7ffed417e000, read_progress_callback=0x7ffed415d720) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:47
#25 0x000000001e8f29e0 in DB::ExecutionThreadContext::executeTask (this=0x7ffe32b49800) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:95
#26 0x000000001e8d7761 in DB::PipelineExecutor::executeStepImpl (this=0x7ffed8deb378, thread_num=0, yield_flag=0x0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:273
#27 0x000000001e8d7a77 in DB::PipelineExecutor::executeSingleThread (this=0x7ffed8deb378, thread_num=0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:239
#28 0x000000001e8d6771 in DB::PipelineExecutor::executeImpl (this=0x7ffed8deb378, num_threads=1, concurrency_control=true) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:399
#29 0x000000001e8d5d76 in DB::PipelineExecutor::execute (this=0x7ffed8deb378, num_threads=1, concurrency_control=true) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:111
#30 0x000000001e8d3daf in DB::CompletedPipelineExecutor::execute (this=0x7ffed8deb7e0) at /ssd/ClickHouse/src/Processors/Executors/CompletedPipelineExecutor.cpp:110
#31 0x000000001d2dcaa0 in DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (DB::QueryResultDetails const&)>, std::__1::optional<DB::FormatSettings> const&, std::__1::function<void (DB::IOutputFormat&)>) (istr=..., ostr=..., allow_into_outfile=false, context=..., set_result_details=..., output_format_settings=..., handle_exception_in_output_format=...) at /ssd/ClickHouse/src/Interpreters/executeQuery.cpp:1448
#32 0x000000001e7b8f91 in DB::HTTPHandler::processQuery (this=0x7ffe46d00020, request=..., params=..., response=..., used_output=..., query_scope=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:884
#33 0x000000001e7bbb67 in DB::HTTPHandler::handleRequest (this=0x7ffe46d00020, request=..., response=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:1078
#34 0x000000001e8644dc in DB::HTTPServerConnection::run (this=0x7ffe46cffc60) at /ssd/ClickHouse/src/Server/HTTP/HTTPServerConnection.cpp:68
#35 0x000000002396f7d9 in Poco::Net::TCPServerConnection::start (this=0x7ffe46cffc60) at /ssd/ClickHouse/base/poco/Net/src/TCPServerConnection.cpp:43
#36 0x000000002397001c in Poco::Net::TCPServerDispatcher::run (this=0x7ffe4aaf4200) at /ssd/ClickHouse/base/poco/Net/src/TCPServerDispatcher.cpp:115
#37 0x0000000023b5ac14 in Poco::PooledThread::run (this=0x7ffff71cab80) at /ssd/ClickHouse/base/poco/Foundation/src/ThreadPool.cpp:188
--Type <RET> for more, q to quit, c to continue without paging--
#38 0x0000000023b579ba in Poco::(anonymous namespace)::RunnableHolder::run (this=0x7ffff70019c0) at /ssd/ClickHouse/base/poco/Foundation/src/Thread.cpp:45
#39 0x0000000023b566be in Poco::ThreadImpl::runnableEntry (pThread=0x7ffff71cabb8) at /ssd/ClickHouse/base/poco/Foundation/src/Thread_POSIX.cpp:335
#40 0x00007ffff7c94ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#41 0x00007ffff7d26a40 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

最后的打开文件路径:

Thread 4 "HTTPHandler" hit Breakpoint 5, __libc_open64 (file=0x7ffe5fcba960 "/var/lib/clickhouse/store/bc6/bc6c6256-174e-49ac-9b85-10b115b137c9/tmp_insert_all_4_4_0/data.cmrk3", oflag=524865) at ../sysdeps/unix/sysv/linux/open64.c:30
30      in ../sysdeps/unix/sysv/linux/open64.c
(gdb) bt
#0  __libc_open64 (file=0x7ffe5fcba960 "/var/lib/clickhouse/store/bc6/bc6c6256-174e-49ac-9b85-10b115b137c9/tmp_insert_all_4_4_0/data.cmrk3", oflag=524865) at ../sysdeps/unix/sysv/linux/open64.c:30
#1  0x0000000013cad0ea in DB::WriteBufferFromFile::WriteBufferFromFile (this=0x7ffe45912ca0, file_name_=..., buf_size=4096, flags=-1, throttler_=..., mode=438, existing_memory=0x0, alignment=0) at /ssd/ClickHouse/src/IO/WriteBufferFromFile.cpp:46
#2  0x000000001bc9c455 in std::__1::make_unique[abi:v15000]<DB::WriteBufferFromFile, std::__1::__fs::filesystem::path, unsigned long&, int&, std::__1::shared_ptr<DB::Throttler> const&>(std::__1::__fs::filesystem::path&&, unsigned long&, int&, std::__1::shared_ptr<DB::Throttler> const&) (__args=..., __args=..., __args=..., __args=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714
#3  0x000000001bc94d7e in DB::DiskLocal::writeFile (this=0x7ffe9721e018, path=..., buf_size=4096, mode=DB::WriteMode::Rewrite, settings=...) at /ssd/ClickHouse/src/Disks/DiskLocal.cpp:342
#4  0x000000001bcafb03 in DB::FakeDiskTransaction::writeFile (this=0x7ffe8900b938, path=..., buf_size=4096, mode=DB::WriteMode::Rewrite, settings=...) at /ssd/ClickHouse/src/Disks/FakeDiskTransaction.h:69
#5  0x000000001df32dd9 in DB::DataPartStorageOnDiskFull::writeFile (this=0x7ffe8d71d418, name=..., buf_size=4096, mode=DB::WriteMode::Rewrite, settings=...) at /ssd/ClickHouse/src/Storages/MergeTree/DataPartStorageOnDiskFull.cpp:132
#6  0x000000001df2fb93 in DB::IDataPartStorage::writeFile (this=0x7ffe8d71d418, name=..., buf_size=4096, settings=...) at /ssd/ClickHouse/src/Storages/MergeTree/IDataPartStorage.h:285
#7  0x000000001e161a22 in DB::MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact (this=0x7ffe5fc70000, data_part_=..., columns_list_=..., metadata_snapshot_=..., indices_to_recalc_=..., marks_file_extension_=..., default_codec_=..., settings_=...,
    index_granularity_=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:40
#8  0x000000001e151d10 in std::__1::make_unique[abi:v15000]<DB::MergeTreeDataPartWriterCompact, std::__1::shared_ptr<DB::IMergeTreeDataPart>, DB::NamesAndTypesList&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::shared_ptr<DB::ICompressionCodec> const&, DB::MergeTreeWriterSettings const&, DB::MergeTreeIndexGranularity const&>(std::__1::shared_ptr<DB::IMergeTreeDataPart>&&, DB::NamesAndTypesList&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&, std::__1::shared_ptr<DB::ICompressionCodec> const&, DB::MergeTreeWriterSettings const&, DB::MergeTreeIndexGranularity const&) (__args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714
#9  0x000000001e14efe0 in DB::MergeTreeDataPartCompact::getWriter (this=0x7ffe8905b618, columns_list=..., metadata_snapshot=..., indices_to_recalc=..., default_codec_=..., writer_settings=..., computed_index_granularity=...)
    at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp:74
#10 0x000000001e3cc967 in DB::MergedBlockOutputStream::MergedBlockOutputStream (this=0x7ffe5fa261c0, data_part=..., metadata_snapshot_=..., columns_list_=..., skip_indices=..., default_codec_=..., txn=..., reset_columns_=false, blocks_are_granules_size=false,
    write_settings_=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergedBlockOutputStream.cpp:50
#11 0x000000001e416327 in std::__1::make_unique[abi:v15000]<DB::MergedBlockOutputStream, std::__1::shared_ptr<DB::IMergeTreeDataPart>&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, DB::NamesAndTypesList&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > >&, std::__1::shared_ptr<DB::ICompressionCodec>&, std::__1::shared_ptr<DB::MergeTreeTransaction>, bool, bool, DB::WriteSettings>(std::__1::shared_ptr<DB::IMergeTreeDataPart>&, std::__1::shared_ptr<DB::StorageInMemoryMetadata const> const&, DB::NamesAndTypesList&, std::__1::vector<std::__1::shared_ptr<DB::IMergeTreeIndex const>, std::__1::allocator<std::__1::shared_ptr<DB::IMergeTreeIndex const> > >&, std::__1::shared_ptr<DB::ICompressionCodec>&, std::__1::shared_ptr<DB::MergeTreeTransaction>&&, bool&&, bool&&, DB::WriteSettings&&) (__args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=..., __args=...)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__memory/unique_ptr.h:714
#12 0x000000001e412851 in DB::MergeTreeDataWriter::writeTempPartImpl (this=0x7ffe732959a0, block_with_partition=..., metadata_snapshot=..., context=..., block_number=4, need_tmp_prefix=true) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:567
#13 0x000000001e40ffdd in DB::MergeTreeDataWriter::writeTempPart (this=0x7ffe732959a0, block=..., metadata_snapshot=..., context=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:385
#14 0x000000001e587b8c in DB::MergeTreeSink::consume (this=0x7ffe5fa3a198, chunk=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeSink.cpp:87
#15 0x000000001ef78c5e in DB::SinkToStorage::onConsume (this=0x7ffe5fa3a198, chunk=...) at /ssd/ClickHouse/src/Processors/Sinks/SinkToStorage.cpp:24
#16 0x000000001ee0a83f in DB::ExceptionKeepingTransform::work()::$_1::operator()() const (this=0x7ffed85e97c8) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#17 0x000000001ee0a7f5 in std::__1::__invoke[abi:v15000]<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__f=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:394
#18 0x000000001ee0a7d5 in std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__args=...)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:479
#19 0x000000001ee0a7b5 in std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>::operator()[abi:v15000]() (this=0x7ffed85e97c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:235
#20 0x000000001ee0a77d in std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()> >(std::__1::__function::__policy_storage const*) (__buf=0x7ffed85e97c8)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:716
#21 0x0000000013ac9556 in std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const (this=0x7ffed85e97c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:848
#22 0x0000000013ac8a35 in std::__1::function<void ()>::operator()() const (this=0x7ffed85e97c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:1187
#23 0x000000001ee0a076 in DB::runStep(std::__1::function<void ()>, DB::ThreadStatus*, std::__1::atomic<unsigned long>*) (step=..., thread_status=0x0, elapsed_ms=0x0) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:114
#24 0x000000001ee09abe in DB::ExceptionKeepingTransform::work (this=0x7ffe5fa3a198) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#25 0x000000001e8f2ca3 in DB::executeJob (node=0x7ffe5fad7900, read_progress_callback=0x7ffe58925620) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:47
#26 0x000000001e8f29e0 in DB::ExecutionThreadContext::executeTask (this=0x7ffe5fad7f00) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:95
#27 0x000000001e8d7761 in DB::PipelineExecutor::executeStepImpl (this=0x7ffed85ea378, thread_num=0, yield_flag=0x0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:273
#28 0x000000001e8d7a77 in DB::PipelineExecutor::executeSingleThread (this=0x7ffed85ea378, thread_num=0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:239
#29 0x000000001e8d6771 in DB::PipelineExecutor::executeImpl (this=0x7ffed85ea378, num_threads=1, concurrency_control=true) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:399
#30 0x000000001e8d5d76 in DB::PipelineExecutor::execute (this=0x7ffed85ea378, num_threads=1, concurrency_control=true) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:111
#31 0x000000001e8d3daf in DB::CompletedPipelineExecutor::execute (this=0x7ffed85ea7e0) at /ssd/ClickHouse/src/Processors/Executors/CompletedPipelineExecutor.cpp:110
#32 0x000000001d2dcaa0 in DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (DB::QueryResultDetails const&)>, std::__1::optional<DB::FormatSettings> const&, std::__1::function<void (DB::IOutputFormat&)>) (istr=..., ostr=..., allow_into_outfile=false, context=..., set_result_details=..., output_format_settings=..., handle_exception_in_output_format=...) at /ssd/ClickHouse/src/Interpreters/executeQuery.cpp:1448
#33 0x000000001e7b8f91 in DB::HTTPHandler::processQuery (this=0x7ffe589254e0, request=..., params=..., response=..., used_output=..., query_scope=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:884
#34 0x000000001e7bbb67 in DB::HTTPHandler::handleRequest (this=0x7ffe589254e0, request=..., response=...) at /ssd/ClickHouse/src/Server/HTTPHandler.cpp:1078
#35 0x000000001e8644dc in DB::HTTPServerConnection::run (this=0x7ffe589253a0) at /ssd/ClickHouse/src/Server/HTTP/HTTPServerConnection.cpp:68
#36 0x000000002396f7d9 in Poco::Net::TCPServerConnection::start (this=0x7ffe589253a0) at /ssd/ClickHouse/base/poco/Net/src/TCPServerConnection.cpp:43
#37 0x000000002397001c in Poco::Net::TCPServerDispatcher::run (this=0x7ffed4e87400) at /ssd/ClickHouse/base/poco/Net/src/TCPServerDispatcher.cpp:115
--Type <RET> for more, q to quit, c to continue without paging--
#38 0x0000000023b5ac14 in Poco::PooledThread::run (this=0x7ffff71cb080) at /ssd/ClickHouse/base/poco/Foundation/src/ThreadPool.cpp:188
#39 0x0000000023b579ba in Poco::(anonymous namespace)::RunnableHolder::run (this=0x7ffff70019e0) at /ssd/ClickHouse/base/poco/Foundation/src/Thread.cpp:45
#40 0x0000000023b566be in Poco::ThreadImpl::runnableEntry (pThread=0x7ffff71cb0b8) at /ssd/ClickHouse/base/poco/Foundation/src/Thread_POSIX.cpp:335
#41 0x00007ffff7c94ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#42 0x00007ffff7d26a40 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

写入bin:

Thread 269 "SystemLogFlush" hit Breakpoint 6, DB::WriteBufferFromFileDescriptor::nextImpl (this=0x7ffe89057680) at /ssd/ClickHouse/src/IO/WriteBufferFromFileDescriptor.cpp:48
48          if (!offset())
(gdb) p fd
$6 = 177
(gdb) bt
#0  DB::WriteBufferFromFileDescriptor::nextImpl (this=0x7ffe89057680) at /ssd/ClickHouse/src/IO/WriteBufferFromFileDescriptor.cpp:48
#1  0x000000000afa964d in DB::WriteBuffer::next (this=0x7ffe89057680) at /ssd/ClickHouse/src/IO/WriteBuffer.h:48
#2  0x000000001df2a9dd in DB::HashingWriteBuffer::nextImpl (this=0x7ffe890142b0) at /ssd/ClickHouse/src/IO/HashingWriteBuffer.h:64
#3  0x000000000afa964d in DB::WriteBuffer::next (this=0x7ffe890142b0) at /ssd/ClickHouse/src/IO/WriteBuffer.h:48
#4  0x0000000013aecd15 in DB::WriteBuffer::finalizeImpl (this=0x7ffe890142b0) at /ssd/ClickHouse/src/IO/WriteBuffer.h:145
#5  0x000000000afa9542 in DB::WriteBuffer::finalize (this=0x7ffe890142b0) at /ssd/ClickHouse/src/IO/WriteBuffer.h:122
#6  0x000000001e16385b in DB::MergeTreeDataPartWriterCompact::fillDataChecksums (this=0x7ffe89014000, checksums=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:295
#7  0x000000001e1641ba in DB::MergeTreeDataPartWriterCompact::fillChecksums (this=0x7ffe89014000, checksums=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp:430
#8  0x000000001e3ccfda in DB::MergedBlockOutputStream::finalizePartAsync (this=0x7ffe89037c40, new_part=..., sync=false, total_columns_list=0x0, additional_column_checksums=0x0) at /ssd/ClickHouse/src/Storages/MergeTree/MergedBlockOutputStream.cpp:151
#9  0x000000001e412cf6 in DB::MergeTreeDataWriter::writeTempPartImpl (this=0x7ffe971d89a0, block_with_partition=..., metadata_snapshot=..., context=..., block_number=620, need_tmp_prefix=true) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:592
#10 0x000000001e40ffdd in DB::MergeTreeDataWriter::writeTempPart (this=0x7ffe971d89a0, block=..., metadata_snapshot=..., context=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeDataWriter.cpp:385
#11 0x000000001e587b8c in DB::MergeTreeSink::consume (this=0x7ffe8906cc58, chunk=...) at /ssd/ClickHouse/src/Storages/MergeTree/MergeTreeSink.cpp:87
#12 0x000000001ef78c5e in DB::SinkToStorage::onConsume (this=0x7ffe8906cc58, chunk=...) at /ssd/ClickHouse/src/Processors/Sinks/SinkToStorage.cpp:24
#13 0x000000001ee0a83f in DB::ExceptionKeepingTransform::work()::$_1::operator()() const (this=0x7ffe4dca61c8) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#14 0x000000001ee0a7f5 in std::__1::__invoke[abi:v15000]<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__f=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:394
#15 0x000000001ee0a7d5 in std::__1::__invoke_void_return_wrapper<void, true>::__call<DB::ExceptionKeepingTransform::work()::$_1&>(DB::ExceptionKeepingTransform::work()::$_1&) (__args=...)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:479
#16 0x000000001ee0a7b5 in std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()>::operator()[abi:v15000]() (this=0x7ffe4dca61c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:235
#17 0x000000001ee0a77d in std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<DB::ExceptionKeepingTransform::work()::$_1, void ()> >(std::__1::__function::__policy_storage const*) (__buf=0x7ffe4dca61c8)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:716
#18 0x0000000013ac9556 in std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const (this=0x7ffe4dca61c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:848
#19 0x0000000013ac8a35 in std::__1::function<void ()>::operator()() const (this=0x7ffe4dca61c8) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:1187
#20 0x000000001ee0a076 in DB::runStep(std::__1::function<void ()>, DB::ThreadStatus*, std::__1::atomic<unsigned long>*) (step=..., thread_status=0x0, elapsed_ms=0x0) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:114
#21 0x000000001ee09abe in DB::ExceptionKeepingTransform::work (this=0x7ffe8906cc58) at /ssd/ClickHouse/src/Processors/Transforms/ExceptionKeepingTransform.cpp:150
#22 0x000000001e8f2ca3 in DB::executeJob (node=0x7ffe890c2400, read_progress_callback=0x7ffe89056b40) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:47
#23 0x000000001e8f29e0 in DB::ExecutionThreadContext::executeTask (this=0x7ffe890c2b00) at /ssd/ClickHouse/src/Processors/Executors/ExecutionThreadContext.cpp:95
#24 0x000000001e8d7761 in DB::PipelineExecutor::executeStepImpl (this=0x7ffe8906dc18, thread_num=0, yield_flag=0x0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:273
#25 0x000000001e8d7259 in DB::PipelineExecutor::executeStep (this=0x7ffe8906dc18, yield_flag=0x0) at /ssd/ClickHouse/src/Processors/Executors/PipelineExecutor.cpp:148
#26 0x000000001e90193e in DB::PushingPipelineExecutor::finish (this=0x7ffe4dca6910) at /ssd/ClickHouse/src/Processors/Executors/PushingPipelineExecutor.cpp:122
#27 0x000000001d0c7e9d in DB::SystemLog<DB::AsynchronousMetricLogElement>::flushImpl (this=0x7ffe70ee4018, to_flush=..., to_flush_end=2074784) at /ssd/ClickHouse/src/Interpreters/SystemLog.cpp:516
#28 0x000000001d0c6e2e in DB::SystemLog<DB::AsynchronousMetricLogElement>::savingThreadFunction (this=0x7ffe70ee4018) at /ssd/ClickHouse/src/Interpreters/SystemLog.cpp:456
#29 0x0000000013c566d9 in DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}::operator()() const (this=0x7ffe4dca77e0) at /ssd/ClickHouse/src/Common/SystemLogBase.cpp:248
#30 0x0000000013c566b5 in std::__1::__invoke[abi:v15000]<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&) (__f=...)
    at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:394
#31 0x0000000013c56699 in std::__1::__apply_tuple_impl[abi:v15000]<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&, std::__1::tuple<>&>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&, std::__1::tuple<>&, std::__1::__tuple_indices<>) (__f=..., __t=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/tuple:1789
#32 0x0000000013c565fd in std::__1::apply[abi:v15000]<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&, std::__1::tuple<>&>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&, std::__1::tuple<>&) (__f=...,
    __t=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/tuple:1798
#33 0x0000000013c56525 in ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&&)::{lambda()#1}::operator()() (this=0x7ffe890fa140) at /ssd/ClickHouse/src/Common/ThreadPool.h:242
#34 0x0000000013c56495 in std::__1::__invoke[abi:v15000]<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&&)::{lambda()#1}&>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&&) (__f=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:394
#35 0x0000000013c56475 in std::__1::__invoke_void_return_wrapper<void, true>::__call<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&&)::{lambda()#1}&>(ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&&)::{lambda()#1}&) (__args=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:479
#36 0x0000000013c56455 in std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&&)::{lambda()#1}, void ()>::operator()[abi:v15000]() (this=0x7ffe890fa140) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:235
#37 0x0000000013c56420 in std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPoolImpl<true>::ThreadFromGlobalPoolImpl<DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}>(DB::SystemLogBase<DB::AsynchronousMetricLogElement>::startup()::{lambda()#1}&&)::{lambda()#1}, void ()> >(std::__1::__function::__policy_storage const*) (__buf=0x7ffe4dca7a98) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:716
#38 0x0000000013ac9556 in std::__1::__function::__policy_func<void ()>::operator()[abi:v15000]() const (this=0x7ffe4dca7a98) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:848
#39 0x0000000013ac8a35 in std::__1::function<void ()>::operator()() const (this=0x7ffe4dca7a98) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/function.h:1187
#40 0x0000000013bd9f23 in ThreadPoolImpl<std::__1::thread>::worker (this=0x7ffff7022100, thread_it=...) at /ssd/ClickHouse/src/Common/ThreadPool.cpp:426
#41 0x0000000013be21c4 in ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::{lambda()#2}::operator()() const (this=0x7ffff6206c48) at /ssd/ClickHouse/src/Common/ThreadPool.cpp:179
--Type <RET> for more, q to quit, c to continue without paging--
#42 0x0000000013be2175 in std::__1::__invoke[abi:v15000]<ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::{lambda()#2}&&) (__f=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/__functional/invoke.h:394
#43 0x0000000013be209d in std::__1::__thread_execute[abi:v15000]<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::{lambda()#2}>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::{lambda()#2}>&, std::__1::__tuple_indices<>) (__t=...) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/thread:284
#44 0x0000000013be1b82 in std::__1::__thread_proxy[abi:v15000]<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, ThreadPoolImpl<std::__1::thread>::scheduleImpl<void>(std::__1::function<void ()>, Priority, std::__1::optional<unsigned long>, bool)::{lambda()#2}> >(void*) (__vp=0x7ffff6206c40) at /ssd/ClickHouse/contrib/llvm-project/libcxx/include/thread:295
#45 0x00007ffff7c94ac3 in start_thread (arg=<optimized out>) at ./nptl/pthread_create.c:442
#46 0x00007ffff7d26a40 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

相关阅读​

clickhouse jdbc 1002 error

· 2 min read

背景​

生产环境会有下面错误: Unknown error 1002, server ClickHouseNode , clickhouse使用的是http的协议

Caused by: java.sql.SQLException: Unknown error 1002, server ClickHouseNode(addr=http:xxxx.amazonaws.com:8123, db=personas)@1267316279
	at com.clickhouse.jdbc.SqlExceptionUtils.handle(SqlExceptionUtils.java:54)
	at com.clickhouse.jdbc.SqlExceptionUtils.handle(SqlExceptionUtils.java:69)
	at com.clickhouse.jdbc.internal.ClickHouseStatementImpl.executeStatement(ClickHouseStatementImpl.java:139)
	at com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.executeBatch(SqlBasedPreparedStatement.java:158)
	at com.clickhouse.jdbc.internal.SqlBasedPreparedStatement.execute(SqlBasedPreparedStatement.java:382)
	at com.zaxxer.hikari.pool.ProxyPreparedStatement.execute(ProxyPreparedStatement.java:44)
	at com.zaxxer.hikari.pool.HikariProxyPreparedStatement.execute(HikariProxyPreparedStatement.java)
	at org.apache.ibatis.executor.statement.PreparedStatementHandler.update(PreparedStatementHandler.java:47)
	at org.apache.ibatis.executor.statement.RoutingStatementHandler.update(RoutingStatementHandler.java:74)
	at org.apache.ibatis.executor.SimpleExecutor.doUpdate(SimpleExecutor.java:50)
	at org.apache.ibatis.executor.BaseExecutor.update(BaseExecutor.java:117)
	at org.apache.ibatis.executor.CachingExecutor.update(CachingExecutor.java:76)
	at jdk.internal.reflect.GeneratedMethodAccessor202.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.apache.ibatis.plugin.Plugin.invoke(Plugin.java:64)
	at com.sun.proxy.$Proxy263.update(Unknown Source)
	at org.apache.ibatis.session.defaults.DefaultSqlSession.update(DefaultSqlSession.java:194)
	at org.apache.ibatis.session.defaults.DefaultSqlSession.insert(DefaultSqlSession.java:181)
	at jdk.internal.reflect.GeneratedMethodAccessor201.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:566)
	at org.mybatis.spring.SqlSessionTemplate$SqlSessionInterceptor.invoke(SqlSessionTemplate.java:427)
	... 18 common frames omitted

翻看源码​

一共有两处会抛出ERROR_UNKNOWN , 代码如下:

// 路径 : com\clickhouse\clickhouse-client\0.3.2\clickhouse-client-0.3.2-sources.jar!\com\clickhouse\client\ClickHouseException.java
    public static final int ERROR_UNKNOWN = 1002;
    private static int extractErrorCode(String errorMessage) {
        if (errorMessage == null || errorMessage.isEmpty()) {
            return ERROR_UNKNOWN;                                     // 抛出unknow
        } else if (errorMessage.startsWith("Poco::Exception. Code: 1000, ")) {
            return ERROR_POCO;
        }

        int startIndex = errorMessage.indexOf(' ');
        if (startIndex >= 0) {
            for (int i = ++startIndex, len = errorMessage.length(); i < len; i++) {
                char ch = errorMessage.charAt(i);
                if (ch == '.' || ch == ',' || Character.isWhitespace(ch)) {
                    try {
                        return Integer.parseInt(errorMessage.substring(startIndex, i));
                    } catch (NumberFormatException e) {
                        // ignore
                    }
                    break;
                }
            }
        }

        // this is confusing as usually it's a client-side exception
        return ERROR_UNKNOWN;                                     // 抛出unknown
    }


看了代码,这个错误外部的异常,不是sql语法错误,所以一般是反向代理一侧有问题,也就是代理层或者clickhouse层有问题

clickhouse mybatis batch insert cpu raise up

· 2 min read

背景​

背景

使用jdbc clickhouse,批量写入,发现cpu升高非常多,升到了90%多

排查及原因​

相关环境​

jdk: jdk11 clickhouse 使用的sql驱动:

               <groupId>com.clickhouse</groupId>
                <artifactId>clickhouse-jdbc</artifactId>
                <version>0.3.2-patch11</version>

原因: 使用mybatis plus 拼写sql,批量写入2000条 sql 用mybatis 的xml 拼写 类似:

insert into table values (row1_field1 , row1_field2 ),(row1_field1 , row1_field2) .... 这里是用mybatis 的xml foreache 2000 次

线上warning日志:

Please consider to use one and only one values expression, for example: use 'values(?)' instead of 'values(?),(?)'.

由于jdbc 的parser 比较慢,需要将perpare语句改成以下形式:

insert into table values ( ?, ? )  ## 只有一次

采用的是

// Note: "insert into table values(?,?)" is treated as "insert into mytable"
try (PreparedStatement ps = conn.prepareStatement("insert into table values(?,?)")) {
    ps.setString(1, "test"); // id
    ps.setObject(2, LocalDateTime.now()); // timestamp
    ps.addBatch(); // append parameters to the query
    ...
    ps.executeBatch(); // issue the composed query: insert into mytable values(...)(...)...(...)
}

结果​

cpu 从80%降低到30%以内 优化前:

优化前

优化后: 优化后

相关阅读​

clickhouse 400 error

· 2 min read

背景​

线上环境请求clickhouse 400 bad request

问题排查​

线上clickhouse导出的是http端口

线上java client抛出400异常:

	ru.yandex.clickhouse.except.ClickHouseException: ClickHouse exception, code: 400, host: internal-LB-clickhouse-write-intranet-429192233.us-west-2.elb.amazonaws.com, port: 8123; HTTP/1.1 400 Bad Request
	at ru.yandex.clickhouse.except.ClickHouseExceptionSpecifier.specify(ClickHouseExceptionSpecifier.java:60)
	at ru.yandex.clickhouse.except.ClickHouseExceptionSpecifier.specify(ClickHouseExceptionSpecifier.java:26)
	at ru.yandex.clickhouse.ClickHouseStatementImpl.checkForErrorAndThrow(ClickHouseStatementImpl.java:1080)
	at ru.yandex.clickhouse.ClickHouseStatementImpl.getInputStream(ClickHouseStatementImpl.java:770)
	at ru.yandex.clickhouse.ClickHouseStatementImpl.getLastInputStream(ClickHouseStatementImpl.java:693)
Caused by: java.lang.IllegalStateException: HTTP/1.1 400 Bad Request
	at ru.yandex.clickhouse.ClickHouseStatementImpl.checkForErrorAndThrow(ClickHouseStatementImpl.java:1080)
	at ru.yandex.clickhouse.ClickHouseStatementImpl.getInputStream(ClickHouseStatementImpl.java:770)
	at ru.yandex.clickhouse.ClickHouseStatementImpl.getLastInputStream(ClickHouseStatementImpl.java:693)
	at ru.yandex.clickhouse.ClickHouseStatementImpl.executeQuery(ClickHouseStatementImpl.java:341)
	at ru.yandex.clickhouse.ClickHouseStatementImpl.executeQuery(ClickHouseStatementImpl.java:326)

打开clickhouse日志,发现有以下报错:

2023.01.12 09:24:26.187479 [ 205027 ] {65179d49-ea6e-4a15-b13b-16d8378cfe29} <Error> executeQuery: Code: 62, e.displayText() = DB::Exception: Syntax error: failed at position 72 ('Client'): Client as DEFAULT_VALUE, Application name as DESCRIPTION union all select CustomHttpHeaders as NAME, toInt32(0) as MAX_LEN,  as DEFAULT_VALUE, Custom HTTP heade. Expected one of: UNION, LIMIT, WHERE, WINDOW, end of query, HAVING, GROUP BY, INTO OUTFILE, OFFSET, PREWHERE, Comma, ORDER BY, SETTINGS, FROM, FORMAT, WITH, token (version 21.3.3.14 (official build)) (from 172.31.42.57:23878) (in query: select ApplicationName as NAME, toInt32(0) as MAX_LEN, ClickHouse Java Client as DEFAULT_VALUE, Application name as DESCRIPTION union all select CustomHttpHeaders as NAME, toInt32(0) as MAX_LEN, as DEFAULT_VALUE, Custom HTTP headers as DESCRIPTION union all select CustomHttpParameters as NAME, toInt32(0) as MAX_LEN, as DEFAULT_VALUE, Customer HTTP query parameters as DESCRIPTION), Stack trace (when copying this message, always include the lines below):

最后在github上找到相关issue: https://github.com/dbeaver/dbeaver/issues/16885 https://github.com/ClickHouse/clickhouse-jdbc/pull/930

发现是clickhouse的jdbc 包的sql错了

ClickHouse Java Client as DEFAULT_VALUE 应该改为 'ClickHouse Java Client' as DEFAULT_VALUE

我们用的jdbc包是0.3.2

            <dependency>
                <groupId>ru.yandex.clickhouse</groupId>
                <artifactId>clickhouse-jdbc</artifactId>
                <version>0.3.2</version>
            </dependency>

将clickhouse jdbc 包改成0.3.2-patch-1 即可. 其实可以用更加新的包,自己选择吧.

xid equal to close_xid

· One min read

背景​

线上环境写入clickhouse的时候出现错误xid equal to close_xid , 只报错一次 ,后续就好了

returned error: Code: 999, e.displayText() = DB::Exception: Cannot allocate block number in ZooKeeper: Coordination::Exception: xid equal to close_xid (Session expired)

排查问题​

看了一下pr , 是19年加的pr ,当xid 等于0xffffffff 的时候就抛出异常,防止出现死锁 . 在异常之后的下一次请求,clickhouse会重新zookpeer,然后重置xid ,保证整个链接正常

相关阅读​

zookeeper connetion loss

· One min read

背景​

returned error: Code: 999, e.displayText() = DB::Exception: Cannot allocate block number in ZooKeeper: Coordination::Exception: Connection loss, path: xxx

场景​

最近偶尔发生clickhouse发生链接丢失的情况

clickhouse parser

· One min read

clickhouse 堆栈

(lldb) bt
* thread #3, name = 'HTTPHandler', stop reason = breakpoint 1.1
  * frame #0: 0x000000001d9c6522 clickhouse-server`DB::tryParseQuery(parser=0x00007fff2d3ef620, _out_query_end=0x00007fff2d3ecc60, all_queries_end="\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZtext/plain; charset=UTF-8", out_error_message="", hilite=false, query_description="", allow_multi_statements=false, max_query_size=262144, max_parser_depth=1000) at parseQuery.cpp:237:32
    frame #1: 0x000000001d9c77bd clickhouse-server`DB::parseQueryAndMovePosition(parser=0x00007fff2d3ef620, pos=0x00007fff2d3ecc60, end="\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZtext/plain; charset=UTF-8", query_description="", allow_multi_statements=false, max_query_size=262144, max_parser_depth=1000) at parseQuery.cpp:343:18
    frame #2: 0x000000001d9c7926 clickhouse-server`DB::parseQuery(parser=0x00007fff2d3ef620, begin="show databases\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZtext/plain; charset=UTF-8", end="\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZtext/plain; charset=UTF-8", query_description="", max_query_size=262144, max_parser_depth=1000) at parseQuery.cpp:360:12
    frame #3: 0x000000001b95ec13 clickhouse-server`DB::executeQueryImpl(begin="show databases\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZtext/plain; charset=UTF-8", end="\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5\xa5ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZtext/plain; charset=UTF-8", context=std::__1::shared_ptr<DB::Context>::element_type @ 0x00007ffff5b23000 strong=4 weak=5, internal=false, stage=Complete, istr=0x00007fff2403d000) at executeQuery.cpp:442:15
    frame #4: 0x000000001b965181 clickhouse-server`DB::executeQuery(istr=0x00007fff2403d000, ostr=0x00007fff240393d8, allow_into_outfile=false, context=std::__1::shared_ptr<DB::Context>::element_type @ 0x00007ffff5b23000 strong=4 weak=5, set_result_details=DB::SetResultDetailsFunc @ 0x00007fff2d3f0bf0, output_format_settings= Has Value=false )>, std::__1::optional<DB::FormatSettings> const&) at executeQuery.cpp:1136:30
    frame #5: 0x000000001d3b7b17 clickhouse-server`DB::HTTPHandler::processQuery(this=0x00007ffff7154d40, request=0x00007fff2d3f2438, params=0x00007fff2d3f1e10, response=0x00007fff2d3f24f0, used_output=0x00007fff2d3f1ec8, query_scope= Has Value=true ) at HTTPHandler.cpp:822:5
    frame #6: 0x000000001d3ba4fe clickhouse-server`DB::HTTPHandler::handleRequest(this=0x00007ffff7154d40, request=0x00007fff2d3f2438, response=0x00007fff2d3f24f0) at HTTPHandler.cpp:960:9
    frame #7: 0x000000001d3f1e17 clickhouse-server`DB::HTTPServerConnection::run(this=0x00007ffff5b18000) at HTTPServerConnection.cpp:65:34
    frame #8: 0x000000002308e1d9 clickhouse-server`Poco::Net::TCPServerConnection::start(this=0x00007ffff5b18000) at TCPServerConnection.cpp:43:3
    frame #9: 0x000000002308e9e6 clickhouse-server`Poco::Net::TCPServerDispatcher::run(this=0x00007fff12875500) at TCPServerDispatcher.cpp:115:20
    frame #10: 0x00000000232cecf4 clickhouse-server`Poco::PooledThread::run(this=0x00007ffff702df80) at ThreadPool.cpp:199:14
    frame #11: 0x00000000232cb81a clickhouse-server`Poco::(anonymous namespace)::RunnableHolder::run(this=0x00007ffff7001330) at Thread.cpp:55:11
    frame #12: 0x00000000232ca5fe clickhouse-server`Poco::ThreadImpl::runnableEntry(pThread=0x00007ffff702dfb8) at Thread_POSIX.cpp:345:27
    frame #13: 0x00007ffff7df8b43 libc.so.6`start_thread(arg=<unavailable>) at pthread_create.c:442:8
    frame #14: 0x00007ffff7e8aa00 libc.so.6`__clone3 at clone3.S:81

clickhouse function

· 5 min read

背景​

想要熟悉clickhouse的内容。

实现​

## /ClickHouse/src/Functions/isNaN.cpp
## 添加一个struct
struct testtrue
{
    static constexpr auto name = "testtrue";    ##函数名
    template <typename T>
    static bool execute(const T t)     ### 执行回调
    {
        /// Suppression for PVS-Studio.
        return true || t;
    }
};
### 起别名
using FunctionTestTure = FunctionNumericPredicate<testtrue>;


void registerFunctionIsNaN(FunctionFactory & factory)
{
    factory.registerFunction<FunctionIsNaN>();
    factory.registerFunction<FunctionTestTure>();  ### 回调注册这个函数
}

重新编译并调用：

SELECT testtrue(2)

Query id: 87e4625d-8b79-4c3a-8153-b333d6a0614f

┌─testtrue(2)─┐
│           1 │
└─────────────┘

注册路径​

(lldb) bt
* thread #1, name = 'clickhouse-serv', stop reason = breakpoint 1.1
  * frame #0: 0x00000000148ee2ac clickhouse-server`DB::registerFunctionIsNaN(factory=0x000000002f984a50) at isNaN.cpp:43:5
    frame #1: 0x0000000010339e2c clickhouse-server`DB::registerFunctionsMiscellaneous(factory=0x000000002f984a50) at registerFunctionsMiscellaneous.cpp:128:5
    frame #2: 0x00000000103391a0 clickhouse-server`DB::registerFunctions() at registerFunctions.cpp:96:5
    frame #3: 0x000000000d063d35 clickhouse-server`DB::Server::main(this=0x00007fffffffd9b8, (null)=size=0) at Server.cpp:623:5
    frame #4: 0x00000000230abec5 clickhouse-server`Poco::Util::Application::run(this=0x00007fffffffd9b8) at Application.cpp:334:8
    frame #5: 0x000000000d06250b clickhouse-server`DB::Server::run(this=0x00007fffffffd9b8) at Server.cpp:461:25
    frame #6: 0x00000000230c6c70 clickhouse-server`Poco::Util::ServerApplication::run(this=0x00007fffffffd9b8, argc=1, argv=0x00007ffff70f7038) at ServerApplication.cpp:611:9
    frame #7: 0x000000000d05f8e1 clickhouse-server`mainEntryClickHouseServer(argc=1, argv=0x00007ffff70f7038) at Server.cpp:187:20
    frame #8: 0x000000000cf7fe63 clickhouse-server`main(argc_=1, argv_=0x00007fffffffdfb8) at main.cpp:409:12
    frame #9: 0x00007ffff7d92d90 libc.so.6`__libc_start_call_main(main=(clickhouse-server`main at main.cpp:380), argc=1, argv=0x00007fffffffdfb8) at libc_start_call_main.h:58:16
    frame #10: 0x00007ffff7d92e40 libc.so.6`__libc_start_main_impl(main=(clickhouse-server`main at main.cpp:380), argc=1, argv=0x00007fffffffdfb8, init=0x00007ffff7ffd040, fini=<unavailable>, rtld_fini=<unavailable>, stack_end=0x00007fffffffdfa8) at libc-start.c:392:3
    frame #11: 0x000000000cf7fb55 clickhouse-server`_start + 37

函数调用路径​

lldb 调试

(lldb) b DB::(anonymous namespace)::testtrue::execute

调用堆栈：

(lldb) bt
* thread #3, name = 'TCPHandler', stop reason = breakpoint 3.1
  * frame #0: 0x00000000148f3dca clickhouse-server`bool DB::(anonymous namespace)::testtrue::execute<char8_t>(t=0x02 u8'\U00000002') at isNaN.cpp:28:9
    frame #1: 0x00000000148f29dd clickhouse-server`COW<DB::IColumn>::immutable_ptr<DB::IColumn> DB::FunctionNumericPredicate<DB::(anonymous namespace)::testtrue>::execute<char8_t>(this=0x00007fff1e44db58, in_untyped=0x00007fff1e447820) const at FunctionNumericPredicate.h:89:31
    frame #2: 0x00000000148f1e8a clickhouse-server`DB::FunctionNumericPredicate<DB::(anonymous namespace)::testtrue>::executeImpl(this=0x00007fff1e44db58, arguments=size=1, (null)=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, (null)=1) const at FunctionNumericPredicate.h:61:22
    frame #3: 0x00000000103a611c clickhouse-server`DB::IFunction::executeImplDryRun(this=0x00007fff1e44db58, arguments=size=1, result_type=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, input_rows_count=1) const at IFunction.h:395:16
    frame #4: 0x00000000103a484d clickhouse-server`DB::FunctionToExecutableFunctionAdaptor::executeDryRunImpl(this=0x00007fff1e44dba0, arguments=size=1, result_type=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, input_rows_count=1) const at IFunctionAdaptors.h:26:26
    frame #5: 0x000000001a8fa3d9 clickhouse-server`DB::IExecutableFunction::executeWithoutLowCardinalityColumns(this=0x00007fff1e44dba0, args=size=1, result_type=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, input_rows_count=1, dry_run=true) const at IFunction.cpp:217:15
    frame #6: 0x000000001a8fa00a clickhouse-server`DB::IExecutableFunction::defaultImplementationForConstantArguments(this=0x00007fff1e44dba0, args=size=1, result_type=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, input_rows_count=1, dry_run=true) const at IFunction.cpp:160:31
    frame #7: 0x000000001a8fa2c4 clickhouse-server`DB::IExecutableFunction::executeWithoutLowCardinalityColumns(this=0x00007fff1e44dba0, args=size=1, result_type=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, input_rows_count=1, dry_run=true) const at IFunction.cpp:209:20
    frame #8: 0x000000001a8faf55 clickhouse-server`DB::IExecutableFunction::executeWithoutSparseColumns(this=0x00007fff1e44dba0, arguments=size=1, result_type=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, input_rows_count=1, dry_run=true) const at IFunction.cpp:267:22
    frame #9: 0x000000001a8fbe43 clickhouse-server`DB::IExecutableFunction::execute(this=0x00007fff1e44dba0, arguments=size=1, result_type=std::__1::shared_ptr<const DB::IDataType>::element_type @ 0x00007fff1e4899c8 strong=2 weak=2, input_rows_count=1, dry_run=true) const at IFunction.cpp:337:16
    frame #10: 0x000000001b02a7b8 clickhouse-server`DB::ActionsDAG::addFunction(this=0x00007fff1e4ac248, function=std::__1::shared_ptr<DB::IFunctionOverloadResolver>::element_type @ 0x00007fff1e44dde0 strong=2 weak=1, children=size=0, result_name="testtrue(2)") at ActionsDAG.cpp:199:37
    frame #11: 0x000000001cdad1a1 clickhouse-server`DB::ScopeStack::addFunction(this=0x00007fff2d1e96f0, function=std::__1::shared_ptr<DB::IFunctionOverloadResolver>::element_type @ 0x00007fff1e44dde0 strong=2 weak=1, argument_names=size=1, result_name="") at ActionsVisitor.cpp:598:51
    frame #12: 0x000000001cdb7485 clickhouse-server`DB::ActionsMatcher::Data::addFunction(this=0x00007fff2d1e9698, function=std::__1::shared_ptr<DB::IFunctionOverloadResolver>::element_type @ 0x00007fff1e44dde0 strong=2 weak=1, argument_names=size=1, result_name=<unavailable>) at ActionsVisitor.h:140:27
    frame #13: 0x000000001cdb0a2b clickhouse-server`DB::ActionsMatcher::visit(node=0x00007fff2a45c9b8, ast=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff2a45c9b8 strong=1 weak=2, data=0x00007fff2d1e9698) at ActionsVisitor.cpp:1093:14
    frame #14: 0x000000001cdad64d clickhouse-server`DB::ActionsMatcher::visit(ast=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff2a45c9b8 strong=1 weak=2, data=0x00007fff2d1e9698) at ActionsVisitor.cpp:655:9
    frame #15: 0x000000001cdb125c clickhouse-server`DB::ActionsMatcher::visit(expression_list=0x00007fff1e4895b8, (null)=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e4895b8 strong=2 weak=2, data=0x00007fff2d1e9698) at ActionsVisitor.cpp:763:17
    frame #16: 0x000000001cdad6b9 clickhouse-server`DB::ActionsMatcher::visit(ast=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e4895b8 strong=2 weak=2, data=0x00007fff2d1e9698) at ActionsVisitor.cpp:659:9
    frame #17: 0x000000001b1ebce5 clickhouse-server`DB::InDepthNodeVisitor<DB::ActionsMatcher, true, false, std::__1::shared_ptr<DB::IAST> const>::visit(this=0x00007fff2d1e9638, ast=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e4895b8 strong=2 weak=2) at InDepthNodeVisitor.h:34:13
    frame #18: 0x000000001b1dc0ea clickhouse-server`DB::ExpressionAnalyzer::getRootActions(this=0x00007fff1e47f780, ast=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e4895b8 strong=2 weak=2, no_makeset_for_subqueries=false, actions=nullptr, only_consts=false) at ExpressionAnalyzer.cpp:587:48
    frame #19: 0x000000001b1e2f9e clickhouse-server`DB::SelectQueryExpressionAnalyzer::appendSelect(this=0x00007fff1e47f780, chain=0x00007fff2d1ea5a0, only_types=false) at ExpressionAnalyzer.cpp:1383:5
    frame #20: 0x000000001b1e6da8 clickhouse-server`DB::ExpressionAnalysisResult::ExpressionAnalysisResult(this=0x00007fff2d1eaa88, query_analyzer=0x00007fff1e47f780, metadata_snapshot=std::__1::shared_ptr<const DB::StorageInMemoryMetadata>::element_type @ 0x00007ffff705a800 strong=4 weak=1, first_stage_=true, second_stage_=true, only_types=false, filter_info_=nullptr, source_header=0x00007fff1e4c9550) at ExpressionAnalyzer.cpp:1830:24
    frame #21: 0x000000001b57ab9d clickhouse-server`DB::InterpreterSelectQuery::getSampleBlockImpl(this=0x00007fff1e4c9000) at InterpreterSelectQuery.cpp:692:23
    frame #22: 0x000000001b5747f9 clickhouse-server`DB::InterpreterSelectQuery::InterpreterSelectQuery(this=0x00007fff2d1ec148, try_move_to_prewhere=true)::$_1::operator()(bool) const at InterpreterSelectQuery.cpp:552:25
    frame #23: 0x000000001b5709f6 clickhouse-server`DB::InterpreterSelectQuery::InterpreterSelectQuery(this=0x00007fff1e4c9000, query_ptr_=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e48e198 strong=1 weak=2, context_=std::__1::shared_ptr<const DB::Context>::element_type @ 0x00007fff1e495000 strong=3 weak=2, input_pipe_= Has Value=false , storage_=nullptr, options_=0x00007fff1e461770, required_result_column_names=size=0, metadata_snapshot_=nullptr, subquery_for_sets_=size=0, prepared_sets_=size=0) at InterpreterSelectQuery.cpp:555:5
    frame #24: 0x000000001b56edf3 clickhouse-server`DB::InterpreterSelectQuery::InterpreterSelectQuery(this=0x00007fff1e4c9000, query_ptr_=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e48e198 strong=1 weak=2, context_=std::__1::shared_ptr<const DB::Context>::element_type @ 0x00007fff1e495000 strong=3 weak=2, options_=0x00007fff1e461770, required_result_column_names_=size=0) at InterpreterSelectQuery.cpp:165:7
    frame #25: 0x000000001b5f4ce5 clickhouse-server`std::__1::__unique_if<DB::InterpreterSelectQuery>::__unique_single std::__1::make_unique<DB::InterpreterSelectQuery, std::__1::shared_ptr<DB::IAST> const&, std::__1::shared_ptr<DB::Context>&, DB::SelectQueryOptions&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&>(__args=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e48e198 strong=1 weak=2, __args=std::__1::shared_ptr<DB::Context>::element_type @ 0x00007fff1e495000 strong=3 weak=2, __args=0x00007fff1e461770, __args=size=0) at unique_ptr.h:725:32
    frame #26: 0x000000001b5f2d09 clickhouse-server`DB::InterpreterSelectWithUnionQuery::buildCurrentChildInterpreter(this=0x00007fff1e461700, ast_ptr_=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e48e198 strong=1 weak=2, current_required_result_column_names=size=0) at InterpreterSelectWithUnionQuery.cpp:223:16
    frame #27: 0x000000001b5f23ed clickhouse-server`DB::InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery(this=0x00007fff1e461700, query_ptr_=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e48f218 strong=2 weak=2, context_=std::__1::shared_ptr<const DB::Context>::element_type @ 0x00007fff1e492800 strong=5 weak=9, options_=0x00007fff2d1ef588, required_result_column_names=size=0) at InterpreterSelectWithUnionQuery.cpp:140:13
    frame #28: 0x000000001b53b830 clickhouse-server`std::__1::__unique_if<DB::InterpreterSelectWithUnionQuery>::__unique_single std::__1::make_unique<DB::InterpreterSelectWithUnionQuery, std::__1::shared_ptr<DB::IAST>&, std::__1::shared_ptr<DB::Context>&, DB::SelectQueryOptions const&>(__args=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e48f218 strong=2 weak=2, __args=std::__1::shared_ptr<DB::Context>::element_type @ 0x00007fff1e492800 strong=5 weak=9, __args=0x00007fff2d1ef588) at unique_ptr.h:725:32
    frame #29: 0x000000001b539e5d clickhouse-server`DB::InterpreterFactory::get(query=std::__1::shared_ptr<DB::IAST>::element_type @ 0x00007fff1e48f218 strong=2 weak=2, context=std::__1::shared_ptr<DB::Context>::element_type @ 0x00007fff1e492800 strong=5 weak=9, options=0x00007fff2d1ef588) at InterpreterFactory.cpp:122:16
    frame #30: 0x000000001b961577 clickhouse-server`DB::executeQueryImpl(begin="select testtrue(2);", end="", context=std::__1::shared_ptr<DB::Context>::element_type @ 0x00007fff1e492800 strong=5 weak=9, internal=false, stage=Complete, istr=0x0000000000000000) at executeQuery.cpp:658:27
    frame #31: 0x000000001b95ee64 clickhouse-server`DB::executeQuery(query="select testtrue(2);", context=std::__1::shared_ptr<DB::Context>::element_type @ 0x00007fff1e492800 strong=5 weak=9, internal=false, stage=Complete) at executeQuery.cpp:1067:30
    frame #32: 0x000000001c590f25 clickhouse-server`DB::TCPHandler::runImpl(this=0x00007fff1e46e000) at TCPHandler.cpp:332:24
    frame #33: 0x000000001c59f9e5 clickhouse-server`DB::TCPHandler::run(this=0x00007fff1e46e000) at TCPHandler.cpp:1781:9
    frame #34: 0x0000000023091f79 clickhouse-server`Poco::Net::TCPServerConnection::start(this=0x00007fff1e46e000) at TCPServerConnection.cpp:43:3
    frame #35: 0x0000000023092786 clickhouse-server`Poco::Net::TCPServerDispatcher::run(this=0x00007fff26648600) at TCPServerDispatcher.cpp:115:20
    frame #36: 0x00000000232d2a94 clickhouse-server`Poco::PooledThread::run(this=0x00007ffff702df80) at ThreadPool.cpp:199:14
    frame #37: 0x00000000232cf5ba clickhouse-server`Poco::(anonymous namespace)::RunnableHolder::run(this=0x00007ffff7001330) at Thread.cpp:55:11
    frame #38: 0x00000000232ce39e clickhouse-server`Poco::ThreadImpl::runnableEntry(pThread=0x00007ffff702dfb8) at Thread_POSIX.cpp:345:27
    frame #39: 0x00007ffff7dfdb43 libc.so.6`start_thread(arg=<unavailable>) at pthread_create.c:442:8
    frame #40: 0x00007ffff7e8fa00 libc.so.6`__clone3 at clone3.S:81

function的实现​

所有函数都是继承IFunction

## ClickHouse/src/Functions/IFunction.h

class IFunction
{
public:

    virtual ~IFunction() = default;

    virtual String getName() const = 0;

    virtual ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const = 0;
    virtual ColumnPtr executeImplDryRun(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
    {
        return executeImpl(arguments, result_type, input_rows_count);
    }
    ...
};

using FunctionPtr = std::shared_ptr<IFunction>;

核心是virtual方法executeImpl

看看实现的模板类

## ClickHouse/src/Functions/FunctionNumericPredicate.h
    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
    {
        const auto * in = arguments.front().column.get();

        ColumnPtr res;
        if (!((res = execute<UInt8>(in))
            || (res = execute<UInt16>(in))
            || (res = execute<UInt32>(in))
            || (res = execute<UInt64>(in))
            || (res = execute<Int8>(in))
            || (res = execute<Int16>(in))
            || (res = execute<Int32>(in))
            || (res = execute<Int64>(in))
            || (res = execute<Float32>(in))
            || (res = execute<Float64>(in))))
            throw Exception{"Illegal column " + in->getName() + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN};

        return res;
    }

    template <typename T>
    ColumnPtr execute(const IColumn * in_untyped) const
    {
        if (const auto in = checkAndGetColumn<ColumnVector<T>>(in_untyped))
        {
            const auto size = in->size();

            auto out = ColumnUInt8::create(size);

            const auto & in_data = in->getData();
            auto & out_data = out->getData();

            for (const auto i : collections::range(0, size))
                out_data[i] = Impl::execute(in_data[i]);

            return out;
        }

        return nullptr;
    }