This commit is contained in:
Yves
2025-02-19 11:20:42 +01:00
parent 80df1f7ce1
commit 5811f237d8
11 changed files with 636 additions and 569 deletions

View File

@@ -3,30 +3,23 @@ cmake_minimum_required(VERSION 3.5...3.31.5)
# Set extension name here # Set extension name here
set(TARGET_NAME ui) set(TARGET_NAME ui)
# DuckDB's extension distribution supports vcpkg. As such, dependencies can be added in ./vcpkg.json and then # DuckDB's extension distribution supports vcpkg. As such, dependencies can be
# used in cmake with find_package. Feel free to remove or replace with other dependencies. # added in ./vcpkg.json and then used in cmake with find_package. Feel free to
# Note that it should also be removed from vcpkg.json to prevent needlessly installing it.. # remove or replace with other dependencies. Note that it should also be removed
# from vcpkg.json to prevent needlessly installing it..
find_package(OpenSSL REQUIRED) find_package(OpenSSL REQUIRED)
set(EXTENSION_NAME ${TARGET_NAME}_extension) set(EXTENSION_NAME ${TARGET_NAME}_extension)
project(${TARGET_NAME}) project(${TARGET_NAME})
include_directories( include_directories(src/include ${DuckDB_SOURCE_DIR}/third_party/httplib)
src/include
${DuckDB_SOURCE_DIR}/third_party/httplib
)
set(EXTENSION_SOURCES set(EXTENSION_SOURCES
src/ui_extension.cpp src/ui_extension.cpp src/http_server.cpp src/utils/encoding.cpp
src/http_server.cpp src/utils/env.cpp src/utils/helpers.cpp src/utils/serialization.cpp)
src/utils/encoding.cpp
src/utils/env.cpp
src/utils/helpers.cpp
src/utils/serialization.cpp
)
find_package(Git) find_package(Git)
if (NOT Git_FOUND) if(NOT Git_FOUND)
message(FATAL_ERROR "Git not found, unable to determine git sha") message(FATAL_ERROR "Git not found, unable to determine git sha")
endif() endif()
@@ -34,8 +27,7 @@ execute_process(
COMMAND ${GIT_EXECUTABLE} rev-parse --short=10 HEAD COMMAND ${GIT_EXECUTABLE} rev-parse --short=10 HEAD
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
OUTPUT_VARIABLE UI_EXTENSION_GIT_SHA OUTPUT_VARIABLE UI_EXTENSION_GIT_SHA
OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_STRIP_TRAILING_WHITESPACE)
)
message(STATUS "UI_EXTENSION_GIT_SHA=${UI_EXTENSION_GIT_SHA}") message(STATUS "UI_EXTENSION_GIT_SHA=${UI_EXTENSION_GIT_SHA}")
add_definitions(-DUI_EXTENSION_GIT_SHA="${UI_EXTENSION_GIT_SHA}") add_definitions(-DUI_EXTENSION_GIT_SHA="${UI_EXTENSION_GIT_SHA}")

View File

@@ -22,8 +22,9 @@ constexpr idx_t EMPTY_SSE_MESSAGE_LENGTH = 3;
bool EventDispatcher::WaitEvent(httplib::DataSink *sink) { bool EventDispatcher::WaitEvent(httplib::DataSink *sink) {
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
// Don't allow too many simultaneous waits, because each consumes a thread in the httplib thread pool, and also // Don't allow too many simultaneous waits, because each consumes a thread in
// browsers limit the number of server-sent event connections. // the httplib thread pool, and also browsers limit the number of server-sent
// event connections.
if (closed_ || wait_count_ >= MAX_EVENT_WAIT_COUNT) { if (closed_ || wait_count_ >= MAX_EVENT_WAIT_COUNT) {
return false; return false;
} }
@@ -67,7 +68,7 @@ void EventDispatcher::Close() {
unique_ptr<HttpServer> HttpServer::instance_; unique_ptr<HttpServer> HttpServer::instance_;
HttpServer* HttpServer::instance() { HttpServer *HttpServer::instance() {
if (!instance_) { if (!instance_) {
instance_ = make_uniq<HttpServer>(); instance_ = make_uniq<HttpServer>();
std::atexit(HttpServer::StopInstance); std::atexit(HttpServer::StopInstance);
@@ -75,9 +76,7 @@ HttpServer* HttpServer::instance() {
return instance_.get(); return instance_.get();
} }
bool HttpServer::Started() { bool HttpServer::Started() { return instance_ && instance_->thread_; }
return instance_ && instance_->thread_;
}
void HttpServer::StopInstance() { void HttpServer::StopInstance() {
if (instance_) { if (instance_) {
@@ -97,7 +96,8 @@ bool HttpServer::Start(const uint16_t local_port, const std::string &remote_url,
#ifndef UI_EXTENSION_GIT_SHA #ifndef UI_EXTENSION_GIT_SHA
#error "UI_EXTENSION_GIT_SHA must be defined" #error "UI_EXTENSION_GIT_SHA must be defined"
#endif #endif
user_agent_ = StringUtil::Format("duckdb-ui/%s(%s)", UI_EXTENSION_GIT_SHA, DuckDB::Platform()); user_agent_ = StringUtil::Format("duckdb-ui/%s(%s)", UI_EXTENSION_GIT_SHA,
DuckDB::Platform());
event_dispatcher_ = make_uniq<EventDispatcher>(); event_dispatcher_ = make_uniq<EventDispatcher>();
thread_ = make_uniq<std::thread>(&HttpServer::Run, this); thread_ = make_uniq<std::thread>(&HttpServer::Run, this);
return true; return true;
@@ -120,9 +120,7 @@ bool HttpServer::Stop() {
return true; return true;
} }
uint16_t HttpServer::LocalPort() { uint16_t HttpServer::LocalPort() { return local_port_; }
return local_port_;
}
void HttpServer::SendConnectedEvent(const std::string &token) { void HttpServer::SendConnectedEvent(const std::string &token) {
SendEvent(StringUtil::Format("event: ConnectedEvent\ndata: %s\n\n", token)); SendEvent(StringUtil::Format("event: ConnectedEvent\ndata: %s\n\n", token));
@@ -140,23 +138,37 @@ void HttpServer::SendEvent(const std::string &message) {
void HttpServer::Run() { void HttpServer::Run() {
server_.Get("/localEvents", server_.Get("/localEvents",
[&](const httplib::Request &req, httplib::Response &res) { HandleGetLocalEvents(req, res); }); [&](const httplib::Request &req, httplib::Response &res) {
HandleGetLocalEvents(req, res);
});
server_.Get("/localToken", server_.Get("/localToken",
[&](const httplib::Request &req, httplib::Response &res) { HandleGetLocalToken(req, res); }); [&](const httplib::Request &req, httplib::Response &res) {
server_.Get("/.*", [&](const httplib::Request &req, httplib::Response &res) { HandleGet(req, res); }); HandleGetLocalToken(req, res);
});
server_.Get("/.*", [&](const httplib::Request &req, httplib::Response &res) {
HandleGet(req, res);
});
server_.Post("/ddb/interrupt", server_.Post("/ddb/interrupt",
[&](const httplib::Request &req, httplib::Response &res) { HandleInterrupt(req, res); }); [&](const httplib::Request &req, httplib::Response &res) {
HandleInterrupt(req, res);
});
server_.Post("/ddb/run", server_.Post("/ddb/run",
[&](const httplib::Request &req, httplib::Response &res, [&](const httplib::Request &req, httplib::Response &res,
const httplib::ContentReader &content_reader) { HandleRun(req, res, content_reader); }); const httplib::ContentReader &content_reader) {
HandleRun(req, res, content_reader);
});
server_.Post("/ddb/tokenize", server_.Post("/ddb/tokenize",
[&](const httplib::Request &req, httplib::Response &res, [&](const httplib::Request &req, httplib::Response &res,
const httplib::ContentReader &content_reader) { HandleTokenize(req, res, content_reader); }); const httplib::ContentReader &content_reader) {
HandleTokenize(req, res, content_reader);
});
server_.listen("localhost", local_port_); server_.listen("localhost", local_port_);
} }
void HttpServer::HandleGetLocalEvents(const httplib::Request &req, httplib::Response &res) { void HttpServer::HandleGetLocalEvents(const httplib::Request &req,
res.set_chunked_content_provider("text/event-stream", [&](size_t /*offset*/, httplib::DataSink &sink) { httplib::Response &res) {
res.set_chunked_content_provider(
"text/event-stream", [&](size_t /*offset*/, httplib::DataSink &sink) {
if (event_dispatcher_->WaitEvent(&sink)) { if (event_dispatcher_->WaitEvent(&sink)) {
return true; return true;
} }
@@ -165,9 +177,11 @@ void HttpServer::HandleGetLocalEvents(const httplib::Request &req, httplib::Resp
}); });
} }
void HttpServer::HandleGetLocalToken(const httplib::Request &req, httplib::Response &res) { void HttpServer::HandleGetLocalToken(const httplib::Request &req,
httplib::Response &res) {
if (!ddb_instance_->ExtensionIsLoaded("motherduck")) { if (!ddb_instance_->ExtensionIsLoaded("motherduck")) {
res.set_content("", "text/plain"); // UI expects an empty response if the extension is not loaded res.set_content("", "text/plain"); // UI expects an empty response if the
// extension is not loaded
return; return;
} }
@@ -175,7 +189,8 @@ void HttpServer::HandleGetLocalToken(const httplib::Request &req, httplib::Respo
auto query_res = connection.Query("CALL get_md_token()"); auto query_res = connection.Query("CALL get_md_token()");
if (query_res->HasError()) { if (query_res->HasError()) {
res.status = 500; res.status = 500;
res.set_content("Could not get token: " + query_res->GetError(), "text/plain"); res.set_content("Could not get token: " + query_res->GetError(),
"text/plain");
return; return;
} }
@@ -185,15 +200,17 @@ void HttpServer::HandleGetLocalToken(const httplib::Request &req, httplib::Respo
res.set_content(token, "text/plain"); res.set_content(token, "text/plain");
} }
void HttpServer::HandleGet(const httplib::Request &req, httplib::Response &res) { void HttpServer::HandleGet(const httplib::Request &req,
httplib::Response &res) {
// Create HTTP client to remote URL // Create HTTP client to remote URL
// TODO: Can this be created once and shared? // TODO: Can this be created once and shared?
httplib::Client client(remote_url_); httplib::Client client(remote_url_);
client.set_keep_alive(true); client.set_keep_alive(true);
// Provide a way to turn on or off server certificate verification, at least for now, because it requires httplib to // Provide a way to turn on or off server certificate verification, at least
// correctly get the root certficates on each platform, which doesn't appear to always work. // for now, because it requires httplib to correctly get the root certficates
// Currently, default to no verification, until we understand when it breaks things. // on each platform, which doesn't appear to always work. Currently, default
// to no verification, until we understand when it breaks things.
if (IsEnvEnabled("ui_enable_server_certificate_verification")) { if (IsEnvEnabled("ui_enable_server_certificate_verification")) {
client.enable_server_certificate_verification(true); client.enable_server_certificate_verification(true);
} else { } else {
@@ -217,7 +234,8 @@ void HttpServer::HandleGet(const httplib::Request &req, httplib::Response &res)
} }
} }
void HttpServer::HandleInterrupt(const httplib::Request &req, httplib::Response &res) { void HttpServer::HandleInterrupt(const httplib::Request &req,
httplib::Response &res) {
auto description = req.get_header_value("X-MD-Description"); auto description = req.get_header_value("X-MD-Description");
auto connection_name = req.get_header_value("X-MD-Connection-Name"); auto connection_name = req.get_header_value("X-MD-Connection-Name");
@@ -235,6 +253,15 @@ void HttpServer::HandleInterrupt(const httplib::Request &req, httplib::Response
void HttpServer::HandleRun(const httplib::Request &req, httplib::Response &res, void HttpServer::HandleRun(const httplib::Request &req, httplib::Response &res,
const httplib::ContentReader &content_reader) { const httplib::ContentReader &content_reader) {
try { try {
DoHandleRun(req, res, content_reader);
} catch (const std::exception &ex) {
SetResponseErrorResult(res, ex.what());
}
}
void HttpServer::DoHandleRun(const httplib::Request &req,
httplib::Response &res,
const httplib::ContentReader &content_reader) {
auto description = req.get_header_value("X-MD-Description"); auto description = req.get_header_value("X-MD-Description");
auto connection_name = req.get_header_value("X-MD-Connection-Name"); auto connection_name = req.get_header_value("X-MD-Connection-Name");
@@ -243,17 +270,18 @@ void HttpServer::HandleRun(const httplib::Request &req, httplib::Response &res,
std::string content = ReadContent(content_reader); std::string content = ReadContent(content_reader);
auto connection = FindOrCreateConnection(connection_name); auto connection = FindOrCreateConnection(connection_name);
// Set current database if optional header is provided. // Set current database if optional header is provided.
if (!database_name.empty()) { if (!database_name.empty()) {
connection->context->RunFunctionInTransaction( connection->context->RunFunctionInTransaction([&] {
[&] { ddb_instance_->GetDatabaseManager().SetDefaultDatabase(*connection->context, database_name); }); ddb_instance_->GetDatabaseManager().SetDefaultDatabase(
*connection->context, database_name);
});
} }
// We use a pending query so we can execute tasks and fetch chunks incrementally. // We use a pending query so we can execute tasks and fetch chunks
// This enables cancellation. // incrementally. This enables cancellation.
unique_ptr<PendingQueryResult> pending; unique_ptr<PendingQueryResult> pending;
// Create pending query, with request content as SQL. // Create pending query, with request content as SQL.
@@ -267,7 +295,8 @@ void HttpServer::HandleRun(const httplib::Request &req, httplib::Response &res,
vector<Value> values; vector<Value> values;
for (idx_t i = 0; i < parameter_count; ++i) { for (idx_t i = 0; i < parameter_count; ++i) {
auto parameter = DecodeBase64(req.get_header_value("X-MD-Parameter", i)); auto parameter = DecodeBase64(req.get_header_value("X-MD-Parameter", i));
values.push_back(Value(parameter)); // TODO: support non-string parameters? (SURF-1546) values.push_back(
Value(parameter)); // TODO: support non-string parameters? (SURF-1546)
} }
pending = prepared->PendingQuery(values, true); pending = prepared->PendingQuery(values, true);
} else { } else {
@@ -291,9 +320,9 @@ void HttpServer::HandleRun(const httplib::Request &req, httplib::Response &res,
switch (exec_result) { switch (exec_result) {
case PendingExecutionResult::EXECUTION_ERROR: { case PendingExecutionResult::EXECUTION_ERROR:
SetResponseErrorResult(res, pending->GetError()); SetResponseErrorResult(res, pending->GetError());
} break; break;
case PendingExecutionResult::EXECUTION_FINISHED: case PendingExecutionResult::EXECUTION_FINISHED:
case PendingExecutionResult::RESULT_READY: { case PendingExecutionResult::RESULT_READY: {
@@ -302,37 +331,35 @@ void HttpServer::HandleRun(const httplib::Request &req, httplib::Response &res,
// Fetch the chunks and serialize the result. // Fetch the chunks and serialize the result.
SuccessResult success_result; SuccessResult success_result;
success_result.column_names_and_types = {std::move(result->names), std::move(result->types)}; success_result.column_names_and_types = {std::move(result->names),
std::move(result->types)};
// TODO: support limiting the number of chunks fetched (SURF-1540) // TODO: support limiting the number of chunks fetched (SURF-1540)
auto chunk = result->Fetch(); auto chunk = result->Fetch();
while (chunk) { while (chunk) {
success_result.chunks.push_back({static_cast<uint16_t>(chunk->size()), std::move(chunk->data)}); success_result.chunks.push_back(
{static_cast<uint16_t>(chunk->size()), std::move(chunk->data)});
chunk = result->Fetch(); chunk = result->Fetch();
} }
MemoryStream success_response_content; MemoryStream success_response_content;
BinarySerializer::Serialize(success_result, success_response_content); BinarySerializer::Serialize(success_result, success_response_content);
SetResponseContent(res, success_response_content); SetResponseContent(res, success_response_content);
} break; break;
default: {
SetResponseErrorResult(res, "Unexpected PendingExecutionResult");
} break;
} }
default:
} catch (const std::exception &ex) { SetResponseErrorResult(res, "Unexpected PendingExecutionResult");
SetResponseErrorResult(res, ex.what()); break;
} }
} }
void HttpServer::HandleTokenize(const httplib::Request &req, httplib::Response &res, void HttpServer::HandleTokenize(const httplib::Request &req,
httplib::Response &res,
const httplib::ContentReader &content_reader) { const httplib::ContentReader &content_reader) {
auto description = req.get_header_value("X-MD-Description"); auto description = req.get_header_value("X-MD-Description");
std::string content = ReadContent(content_reader); std::string content = ReadContent(content_reader);
auto tokens = Parser::Tokenize(content); auto tokens = Parser::Tokenize(content);
// Read and serialize result // Read and serialize result
@@ -350,7 +377,8 @@ void HttpServer::HandleTokenize(const httplib::Request &req, httplib::Response &
SetResponseContent(res, response_content); SetResponseContent(res, response_content);
} }
std::string HttpServer::ReadContent(const httplib::ContentReader &content_reader) { std::string
HttpServer::ReadContent(const httplib::ContentReader &content_reader) {
std::ostringstream oss; std::ostringstream oss;
content_reader([&](const char *data, size_t data_length) { content_reader([&](const char *data, size_t data_length) {
oss.write(data, data_length); oss.write(data, data_length);
@@ -359,12 +387,14 @@ std::string HttpServer::ReadContent(const httplib::ContentReader &content_reader
return oss.str(); return oss.str();
} }
shared_ptr<Connection> HttpServer::FindConnection(const std::string &connection_name) { shared_ptr<Connection>
HttpServer::FindConnection(const std::string &connection_name) {
if (connection_name.empty()) { if (connection_name.empty()) {
return nullptr; return nullptr;
} }
// Need to protect access to the connections map because this can be called from multiple threads. // Need to protect access to the connections map because this can be called
// from multiple threads.
std::lock_guard<std::mutex> guard(connections_mutex_); std::lock_guard<std::mutex> guard(connections_mutex_);
auto result = connections_.find(connection_name); auto result = connections_.find(connection_name);
@@ -375,13 +405,16 @@ shared_ptr<Connection> HttpServer::FindConnection(const std::string &connection_
return nullptr; return nullptr;
} }
shared_ptr<Connection> HttpServer::FindOrCreateConnection(const std::string &connection_name) { shared_ptr<Connection>
HttpServer::FindOrCreateConnection(const std::string &connection_name) {
if (connection_name.empty()) { if (connection_name.empty()) {
// If no connection name was provided, create and return a new connection but don't remember it. // If no connection name was provided, create and return a new connection
// but don't remember it.
return make_shared_ptr<Connection>(*ddb_instance_); return make_shared_ptr<Connection>(*ddb_instance_);
} }
// Need to protect access to the connections map because this can be called from multiple threads. // Need to protect access to the connections map because this can be called
// from multiple threads.
std::lock_guard<std::mutex> guard(connections_mutex_); std::lock_guard<std::mutex> guard(connections_mutex_);
// If an existing connection with the provided name was found, return it. // If an existing connection with the provided name was found, return it.
@@ -396,10 +429,12 @@ shared_ptr<Connection> HttpServer::FindOrCreateConnection(const std::string &con
return connection; return connection;
} }
void HttpServer::SetResponseContent(httplib::Response &res, const MemoryStream &content) { void HttpServer::SetResponseContent(httplib::Response &res,
const MemoryStream &content) {
auto data = content.GetData(); auto data = content.GetData();
auto length = content.GetPosition(); auto length = content.GetPosition();
res.set_content(reinterpret_cast<const char *>(data), length, "application/octet-stream"); res.set_content(reinterpret_cast<const char *>(data), length,
"application/octet-stream");
} }
void HttpServer::SetResponseEmptyResult(httplib::Response &res) { void HttpServer::SetResponseEmptyResult(httplib::Response &res) {
@@ -409,7 +444,8 @@ void HttpServer::SetResponseEmptyResult(httplib::Response &res) {
SetResponseContent(res, response_content); SetResponseContent(res, response_content);
} }
void HttpServer::SetResponseErrorResult(httplib::Response &res, const std::string &error) { void HttpServer::SetResponseErrorResult(httplib::Response &res,
const std::string &error) {
ErrorResult error_result; ErrorResult error_result;
error_result.error = error; error_result.error = error;
MemoryStream response_content; MemoryStream response_content;
@@ -418,4 +454,4 @@ void HttpServer::SetResponseErrorResult(httplib::Response &res, const std::strin
} }
} // namespace ui } // namespace ui
} // namespace md } // namespace duckdb

View File

@@ -27,17 +27,17 @@ public:
private: private:
std::mutex mutex_; std::mutex mutex_;
std::condition_variable cv_; std::condition_variable cv_;
std::atomic_int next_id_ {0}; std::atomic_int next_id_{0};
std::atomic_int current_id_ {-1}; std::atomic_int current_id_{-1};
std::atomic_int wait_count_ {0}; std::atomic_int wait_count_{0};
std::string message_; std::string message_;
std::atomic_bool closed_ {false}; std::atomic_bool closed_{false};
}; };
class HttpServer { class HttpServer {
public: public:
static HttpServer* instance(); static HttpServer *instance();
static bool Started(); static bool Started();
static void StopInstance(); static void StopInstance();
@@ -51,16 +51,21 @@ public:
private: private:
void SendEvent(const std::string &message); void SendEvent(const std::string &message);
void Run(); void Run();
void HandleGetLocalEvents(const httplib::Request &req, httplib::Response &res); void HandleGetLocalEvents(const httplib::Request &req,
httplib::Response &res);
void HandleGetLocalToken(const httplib::Request &req, httplib::Response &res); void HandleGetLocalToken(const httplib::Request &req, httplib::Response &res);
void HandleGet(const httplib::Request &req, httplib::Response &res); void HandleGet(const httplib::Request &req, httplib::Response &res);
void HandleInterrupt(const httplib::Request &req, httplib::Response &res); void HandleInterrupt(const httplib::Request &req, httplib::Response &res);
void HandleRun(const httplib::Request &req, httplib::Response &res, const httplib::ContentReader &contentReader); void DoHandleRun(const httplib::Request &req, httplib::Response &res,
const httplib::ContentReader &contentReader);
void HandleRun(const httplib::Request &req, httplib::Response &res,
const httplib::ContentReader &contentReader);
void HandleTokenize(const httplib::Request &req, httplib::Response &res, void HandleTokenize(const httplib::Request &req, httplib::Response &res,
const httplib::ContentReader &contentReader); const httplib::ContentReader &contentReader);
std::string ReadContent(const httplib::ContentReader &contentReader); std::string ReadContent(const httplib::ContentReader &contentReader);
shared_ptr<Connection> FindConnection(const std::string &connectionName); shared_ptr<Connection> FindConnection(const std::string &connectionName);
shared_ptr<Connection> FindOrCreateConnection(const std::string &connectionName); shared_ptr<Connection>
FindOrCreateConnection(const std::string &connectionName);
void SetResponseContent(httplib::Response &res, const MemoryStream &content); void SetResponseContent(httplib::Response &res, const MemoryStream &content);
void SetResponseEmptyResult(httplib::Response &res); void SetResponseEmptyResult(httplib::Response &res);
void SetResponseErrorResult(httplib::Response &res, const std::string &error); void SetResponseErrorResult(httplib::Response &res, const std::string &error);
@@ -76,7 +81,8 @@ private:
unique_ptr<EventDispatcher> event_dispatcher_; unique_ptr<EventDispatcher> event_dispatcher_;
static unique_ptr<HttpServer> instance_; static unique_ptr<HttpServer> instance_;
};; };
;
} // namespace ui } // namespace ui
} // namespace duckdb } // namespace duckdb

View File

@@ -6,10 +6,10 @@
namespace duckdb { namespace duckdb {
typedef std::string (*simple_tf_t) (ClientContext &); typedef std::string (*simple_tf_t)(ClientContext &);
struct RunOnceTableFunctionState : GlobalTableFunctionState { struct RunOnceTableFunctionState : GlobalTableFunctionState {
RunOnceTableFunctionState() : run(false) {}; RunOnceTableFunctionState() : run(false){};
std::atomic<bool> run; std::atomic<bool> run;
static unique_ptr<GlobalTableFunctionState> Init(ClientContext &, static unique_ptr<GlobalTableFunctionState> Init(ClientContext &,
@@ -19,9 +19,11 @@ struct RunOnceTableFunctionState : GlobalTableFunctionState {
}; };
template <typename T> template <typename T>
T GetSetting(const ClientContext &context, const char *setting_name, const T default_value) { T GetSetting(const ClientContext &context, const char *setting_name,
const T default_value) {
Value value; Value value;
return context.TryGetCurrentSetting(setting_name, value) ? value.GetValue<T>() : default_value; return context.TryGetCurrentSetting(setting_name, value) ? value.GetValue<T>()
: default_value;
} }
namespace internal { namespace internal {
@@ -31,57 +33,68 @@ unique_ptr<FunctionData> ResultBind(ClientContext &, TableFunctionBindInput &,
bool ShouldRun(TableFunctionInput &input); bool ShouldRun(TableFunctionInput &input);
template <typename Func> template <typename Func> struct CallFunctionHelper;
struct CallFunctionHelper;
template <> template <> struct CallFunctionHelper<std::string (*)()> {
struct CallFunctionHelper<std::string(*)()> { static std::string call(ClientContext &context, TableFunctionInput &input,
static std::string call(ClientContext &context, TableFunctionInput &input, std::string(*f)()) { std::string (*f)()) {
return f(); return f();
} }
}; };
template <> template <> struct CallFunctionHelper<std::string (*)(ClientContext &)> {
struct CallFunctionHelper<std::string(*)(ClientContext &)> { static std::string call(ClientContext &context, TableFunctionInput &input,
static std::string call(ClientContext &context, TableFunctionInput &input, std::string(*f)(ClientContext &)) { std::string (*f)(ClientContext &)) {
return f(context); return f(context);
} }
}; };
template <> template <>
struct CallFunctionHelper<std::string(*)(ClientContext &, TableFunctionInput &)> { struct CallFunctionHelper<std::string (*)(ClientContext &,
static std::string call(ClientContext &context, TableFunctionInput &input, std::string(*f)(ClientContext &, TableFunctionInput &)) { TableFunctionInput &)> {
static std::string call(ClientContext &context, TableFunctionInput &input,
std::string (*f)(ClientContext &,
TableFunctionInput &)) {
return f(context, input); return f(context, input);
} }
}; };
template <typename Func, Func func> template <typename Func, Func func>
void TableFunc(ClientContext &context, TableFunctionInput &input, DataChunk &output) { void TableFunc(ClientContext &context, TableFunctionInput &input,
DataChunk &output) {
if (!ShouldRun(input)) { if (!ShouldRun(input)) {
return; return;
} }
const std::string result = CallFunctionHelper<Func>::call(context, input, func); const std::string result =
CallFunctionHelper<Func>::call(context, input, func);
output.SetCardinality(1); output.SetCardinality(1);
output.SetValue(0, 0, result); output.SetValue(0, 0, result);
} }
template <typename Func, Func func> template <typename Func, Func func>
void RegisterTF(DatabaseInstance &instance, const char* name) { void RegisterTF(DatabaseInstance &instance, const char *name) {
TableFunction tf(name, {}, internal::TableFunc<Func, func>, internal::ResultBind, RunOnceTableFunctionState::Init); TableFunction tf(name, {}, internal::TableFunc<Func, func>,
internal::ResultBind, RunOnceTableFunctionState::Init);
ExtensionUtil::RegisterFunction(instance, tf); ExtensionUtil::RegisterFunction(instance, tf);
} }
template <typename Func, Func func> template <typename Func, Func func>
void RegisterTFWithArgs(DatabaseInstance &instance, const char* name, vector<LogicalType> arguments, table_function_bind_t bind) { void RegisterTFWithArgs(DatabaseInstance &instance, const char *name,
TableFunction tf(name, arguments, internal::TableFunc<Func, func>, bind, RunOnceTableFunctionState::Init); vector<LogicalType> arguments,
table_function_bind_t bind) {
TableFunction tf(name, arguments, internal::TableFunc<Func, func>, bind,
RunOnceTableFunctionState::Init);
ExtensionUtil::RegisterFunction(instance, tf); ExtensionUtil::RegisterFunction(instance, tf);
} }
} } // namespace internal
#define RESISTER_TF(name, func) internal::RegisterTF<decltype(&func), &func>(instance, name) #define RESISTER_TF(name, func) \
internal::RegisterTF<decltype(&func), &func>(instance, name)
#define RESISTER_TF_ARGS(name, args, func, bind) internal::RegisterTFWithArgs<decltype(&func), &func>(instance, name, args, bind) #define RESISTER_TF_ARGS(name, args, func, bind) \
internal::RegisterTFWithArgs<decltype(&func), &func>(instance, name, args, \
bind)
} // namespace duckdb } // namespace duckdb

View File

@@ -16,11 +16,13 @@
#endif #endif
#define UI_LOCAL_PORT_SETTING_NAME "ui_local_port" #define UI_LOCAL_PORT_SETTING_NAME "ui_local_port"
#define UI_LOCAL_PORT_SETTING_DESCRIPTION "Local port on which the UI server listens" #define UI_LOCAL_PORT_SETTING_DESCRIPTION \
"Local port on which the UI server listens"
#define UI_LOCAL_PORT_SETTING_DEFAULT 4213 #define UI_LOCAL_PORT_SETTING_DEFAULT 4213
#define UI_REMOTE_URL_SETTING_NAME "ui_remote_url" #define UI_REMOTE_URL_SETTING_NAME "ui_remote_url"
#define UI_REMOTE_URL_SETTING_DESCRIPTION "Remote URL to which the UI server forwards GET requests" #define UI_REMOTE_URL_SETTING_DESCRIPTION \
"Remote URL to which the UI server forwards GET requests"
#define UI_REMOTE_URL_SETTING_DEFAULT "https://app.motherduck.com" #define UI_REMOTE_URL_SETTING_DEFAULT "https://app.motherduck.com"
namespace duckdb { namespace duckdb {
@@ -28,14 +30,19 @@ namespace duckdb {
namespace internal { namespace internal {
bool StartHttpServer(const ClientContext &context) { bool StartHttpServer(const ClientContext &context) {
const auto url = GetSetting<std::string>(context, UI_REMOTE_URL_SETTING_NAME, const auto url =
GetEnvOrDefault(UI_REMOTE_URL_SETTING_NAME, UI_REMOTE_URL_SETTING_DEFAULT)); GetSetting<std::string>(context, UI_REMOTE_URL_SETTING_NAME,
const uint16_t port = GetSetting(context, UI_LOCAL_PORT_SETTING_NAME, UI_LOCAL_PORT_SETTING_DEFAULT);; GetEnvOrDefault(UI_REMOTE_URL_SETTING_NAME,
UI_REMOTE_URL_SETTING_DEFAULT));
const uint16_t port = GetSetting(context, UI_LOCAL_PORT_SETTING_NAME,
UI_LOCAL_PORT_SETTING_DEFAULT);
;
return ui::HttpServer::instance()->Start(port, url, context.db); return ui::HttpServer::instance()->Start(port, url, context.db);
} }
std::string GetHttpServerLocalURL() { std::string GetHttpServerLocalURL() {
return StringUtil::Format("http://localhost:%d/", ui::HttpServer::instance()->LocalPort()); return StringUtil::Format("http://localhost:%d/",
ui::HttpServer::instance()->LocalPort());
} }
} // namespace internal } // namespace internal
@@ -44,19 +51,23 @@ std::string StartUIFunction(ClientContext &context) {
internal::StartHttpServer(context); internal::StartHttpServer(context);
auto local_url = internal::GetHttpServerLocalURL(); auto local_url = internal::GetHttpServerLocalURL();
const std::string command = StringUtil::Format("%s %s", OPEN_COMMAND, local_url); const std::string command =
return system(command.c_str()) ? StringUtil::Format("%s %s", OPEN_COMMAND, local_url);
StringUtil::Format("Navigate browser to %s", local_url) // open command failed return system(command.c_str())
? StringUtil::Format("Navigate browser to %s",
local_url) // open command failed
: StringUtil::Format("MotherDuck UI started at %s", local_url); : StringUtil::Format("MotherDuck UI started at %s", local_url);
} }
std::string StartUIServerFunction(ClientContext &context) { std::string StartUIServerFunction(ClientContext &context) {
const char* already = internal::StartHttpServer(context) ? "already " : ""; const char *already = internal::StartHttpServer(context) ? "already " : "";
return StringUtil::Format("MotherDuck UI server %sstarted at %s", already, internal::GetHttpServerLocalURL()); return StringUtil::Format("MotherDuck UI server %sstarted at %s", already,
internal::GetHttpServerLocalURL());
} }
std::string StopUIServerFunction() { std::string StopUIServerFunction() {
return ui::HttpServer::instance()->Stop() ? "UI server stopped" : "UI server already stopped"; return ui::HttpServer::instance()->Stop() ? "UI server stopped"
: "UI server already stopped";
} }
// Connected notification // Connected notification
@@ -66,7 +77,8 @@ struct NotifyConnectedFunctionData : public TableFunctionData {
std::string token; std::string token;
}; };
static unique_ptr<FunctionData> NotifyConnectedBind(ClientContext &, TableFunctionBindInput &input, static unique_ptr<FunctionData>
NotifyConnectedBind(ClientContext &, TableFunctionBindInput &input,
vector<LogicalType> &out_types, vector<string> &out_names) { vector<LogicalType> &out_types, vector<string> &out_names) {
if (input.inputs[0].IsNull()) { if (input.inputs[0].IsNull()) {
throw BinderException("Must provide a token"); throw BinderException("Must provide a token");
@@ -77,7 +89,8 @@ static unique_ptr<FunctionData> NotifyConnectedBind(ClientContext &, TableFuncti
return make_uniq<NotifyConnectedFunctionData>(input.inputs[0].ToString()); return make_uniq<NotifyConnectedFunctionData>(input.inputs[0].ToString());
} }
std::string NotifyConnectedFunction(ClientContext &context, TableFunctionInput &input) { std::string NotifyConnectedFunction(ClientContext &context,
TableFunctionInput &input) {
auto &inputs = input.bind_data->Cast<NotifyConnectedFunctionData>(); auto &inputs = input.bind_data->Cast<NotifyConnectedFunctionData>();
ui::HttpServer::instance()->SendConnectedEvent(inputs.token); ui::HttpServer::instance()->SendConnectedEvent(inputs.token);
return "OK"; return "OK";
@@ -93,26 +106,26 @@ std::string NotifyCatalogChangedFunction() {
static void LoadInternal(DatabaseInstance &instance) { static void LoadInternal(DatabaseInstance &instance) {
auto &config = DBConfig::GetConfig(instance); auto &config = DBConfig::GetConfig(instance);
config.AddExtensionOption(UI_LOCAL_PORT_SETTING_NAME, UI_LOCAL_PORT_SETTING_DESCRIPTION, config.AddExtensionOption(
UI_LOCAL_PORT_SETTING_NAME, UI_LOCAL_PORT_SETTING_DESCRIPTION,
LogicalType::USMALLINT, Value::USMALLINT(UI_LOCAL_PORT_SETTING_DEFAULT)); LogicalType::USMALLINT, Value::USMALLINT(UI_LOCAL_PORT_SETTING_DEFAULT));
config.AddExtensionOption( config.AddExtensionOption(
UI_REMOTE_URL_SETTING_NAME, UI_REMOTE_URL_SETTING_DESCRIPTION, LogicalType::VARCHAR, UI_REMOTE_URL_SETTING_NAME, UI_REMOTE_URL_SETTING_DESCRIPTION,
Value(GetEnvOrDefault(UI_REMOTE_URL_SETTING_NAME, UI_REMOTE_URL_SETTING_DEFAULT))); LogicalType::VARCHAR,
Value(GetEnvOrDefault(UI_REMOTE_URL_SETTING_NAME,
UI_REMOTE_URL_SETTING_DEFAULT)));
RESISTER_TF("start_ui", StartUIFunction); RESISTER_TF("start_ui", StartUIFunction);
RESISTER_TF("start_ui_server", StartUIServerFunction); RESISTER_TF("start_ui_server", StartUIServerFunction);
RESISTER_TF("stop_ui_server", StopUIServerFunction); RESISTER_TF("stop_ui_server", StopUIServerFunction);
RESISTER_TF("notify_ui_catalog_changed", NotifyCatalogChangedFunction); RESISTER_TF("notify_ui_catalog_changed", NotifyCatalogChangedFunction);
RESISTER_TF_ARGS("notify_ui_connected", {LogicalType::VARCHAR}, NotifyConnectedFunction, NotifyConnectedBind); RESISTER_TF_ARGS("notify_ui_connected", {LogicalType::VARCHAR},
NotifyConnectedFunction, NotifyConnectedBind);
} }
void UiExtension::Load(DuckDB &db) { void UiExtension::Load(DuckDB &db) { LoadInternal(*db.instance); }
LoadInternal(*db.instance); std::string UiExtension::Name() { return "ui"; }
}
std::string UiExtension::Name() {
return "ui";
}
std::string UiExtension::Version() const { std::string UiExtension::Version() const {
#ifdef UI_EXTENSION_GIT_SHA #ifdef UI_EXTENSION_GIT_SHA

View File

@@ -5,8 +5,10 @@
namespace duckdb { namespace duckdb {
// Copied from https://www.mycplus.com/source-code/c-source-code/base64-encode-decode/ // Copied from
constexpr char k_encoding_table[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+_"; // https://www.mycplus.com/source-code/c-source-code/base64-encode-decode/
constexpr char k_encoding_table[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+_";
std::vector<char> BuildDecodingTable() { std::vector<char> BuildDecodingTable() {
std::vector<char> decoding_table; std::vector<char> decoding_table;
@@ -42,7 +44,8 @@ std::string DecodeBase64(const std::string &data) {
uint32_t sextet_c = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]]; uint32_t sextet_c = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]];
uint32_t sextet_d = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]]; uint32_t sextet_d = data[i] == '=' ? 0 & i++ : k_decoding_table[data[i++]];
uint32_t triple = (sextet_a << 3 * 6) + (sextet_b << 2 * 6) + (sextet_c << 1 * 6) + (sextet_d << 0 * 6); uint32_t triple = (sextet_a << 3 * 6) + (sextet_b << 2 * 6) +
(sextet_c << 1 * 6) + (sextet_d << 0 * 6);
if (j < output_length) { if (j < output_length) {
decoded_data[j++] = (triple >> 2 * 8) & 0xFF; decoded_data[j++] = (triple >> 2 * 8) & 0xFF;

View File

@@ -4,7 +4,8 @@ namespace duckdb {
namespace internal { namespace internal {
bool ShouldRun(TableFunctionInput &input) { bool ShouldRun(TableFunctionInput &input) {
auto state = dynamic_cast<RunOnceTableFunctionState *>(input.global_state.get()); auto state =
dynamic_cast<RunOnceTableFunctionState *>(input.global_state.get());
D_ASSERT(state != nullptr); D_ASSERT(state != nullptr);
if (state->run) { if (state->run) {
return false; return false;

View File

@@ -6,8 +6,7 @@
namespace duckdb { namespace duckdb {
namespace ui { namespace ui {
void EmptyResult::Serialize(Serializer &) const { void EmptyResult::Serialize(Serializer &) const {}
}
void TokenizeResult::Serialize(Serializer &serializer) const { void TokenizeResult::Serialize(Serializer &serializer) const {
serializer.WriteProperty(100, "offsets", offsets); serializer.WriteProperty(100, "offsets", offsets);
@@ -23,9 +22,11 @@ void ColumnNamesAndTypes::Serialize(Serializer &serializer) const {
// Adapted from parts of DataChunk::Serialize // Adapted from parts of DataChunk::Serialize
void Chunk::Serialize(Serializer &serializer) const { void Chunk::Serialize(Serializer &serializer) const {
serializer.WriteProperty(100, "row_count", row_count); serializer.WriteProperty(100, "row_count", row_count);
serializer.WriteList(101, "vectors", vectors.size(), [&](Serializer::List &list, idx_t i) { serializer.WriteList(101, "vectors", vectors.size(),
[&](Serializer::List &list, idx_t i) {
list.WriteObject([&](Serializer &object) { list.WriteObject([&](Serializer &object) {
// Reference the vector to avoid potentially mutating it during serialization // Reference the vector to avoid potentially mutating
// it during serialization
Vector serialized_vector(vectors[i].GetType()); Vector serialized_vector(vectors[i].GetType());
serialized_vector.Reference(vectors[i]); serialized_vector.Reference(vectors[i]);
serialized_vector.Serialize(object, row_count); serialized_vector.Serialize(object, row_count);
@@ -35,8 +36,10 @@ void Chunk::Serialize(Serializer &serializer) const {
void SuccessResult::Serialize(Serializer &serializer) const { void SuccessResult::Serialize(Serializer &serializer) const {
serializer.WriteProperty(100, "success", true); serializer.WriteProperty(100, "success", true);
serializer.WriteProperty(101, "column_names_and_types", column_names_and_types); serializer.WriteProperty(101, "column_names_and_types",
serializer.WriteList(102, "chunks", chunks.size(), column_names_and_types);
serializer.WriteList(
102, "chunks", chunks.size(),
[&](Serializer::List &list, idx_t i) { list.WriteElement(chunks[i]); }); [&](Serializer::List &list, idx_t i) { list.WriteElement(chunks[i]); });
} }