Check format

Add support for new extension loader

.github/workflows/MainDistributionPipeline.yml

@@ -3,8 +3,14 @@
 #
 name: Main Extension Distribution Pipeline
 on:
-  push:
   pull_request:
+    paths-ignore:
+      - "ts/**"
+  push:
+    branches:
+      - "main"
+    paths-ignore:
+      - "ts/**"
   workflow_dispatch:
 
 concurrency:
@@ -23,31 +29,43 @@ jobs:
 
   duckdb-next-patch-build:
     name: Build next patch extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.3.0
     with:
-      ci_tools_version: v1.2.1
+      ci_tools_version: v1.3.0
-      duckdb_version: v1.2-histrionicus
+      duckdb_version: v1.3-ossivalis
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
       extension_name: ui
 
   duckdb-stable-build:
     name: Build stable extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.3.0
     with:
-      ci_tools_version: v1.2.1
+      ci_tools_version: v1.3.0
-      duckdb_version: v1.2.1
+      duckdb_version: v1.3.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
       extension_name: ui
 
   duckdb-stable-deploy:
-    if: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
+    if: ${{ github.repository == 'duckdb/duckdb-ui' && ( startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' ) }}
     name: Deploy stable extension binaries
     needs: duckdb-stable-build
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.2.1
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.3.0
     secrets: inherit
     with:
       extension_name: ui
-      duckdb_version: v1.2.1
+      ci_tools_version: v1.3.0
-      ci_tools_version: v1.2.1
+      duckdb_version: v1.3.0
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads'
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
+
+  format-check:
+    name: Check formatting
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Check formatting
+        run: make format-check

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "duckdb"]
 	path = duckdb
 	url = https://github.com/duckdb/duckdb
-	branch = v1.2.1
+	branch = v1.3.0
 [submodule "extension-ci-tools"]
 	path = extension-ci-tools
 	url = https://github.com/duckdb/extension-ci-tools

src/http_server.cpp

@@ -9,6 +9,7 @@
 #include "utils/serialization.hpp"
 #include "version.hpp"
 #include "watcher.hpp"
+#include <duckdb/common/http_util.hpp>
 #include <duckdb/common/serializer/binary_serializer.hpp>
 #include <duckdb/common/serializer/memory_stream.hpp>
 #include <duckdb/main/attached_database.hpp>
@@ -94,13 +95,17 @@ const HttpServer &HttpServer::Start(ClientContext &context, bool *was_started) {
 
   const auto remote_url = GetRemoteUrl(context);
   const auto port = GetLocalPort(context);
+  auto &http_util = HTTPUtil::Get(*context.db);
+  // FIXME - https://github.com/duckdb/duckdb/pull/17655 will remove `unused`
+  auto http_params = http_util.InitializeParameters(context, "unused");
   auto server = GetInstance(context);
-  server->DoStart(port, remote_url);
+  server->DoStart(port, remote_url, std::move(http_params));
   return *server;
 }
 
 void HttpServer::DoStart(const uint16_t _local_port,
-                         const std::string &_remote_url) {
+                         const std::string &_remote_url,
+                         unique_ptr<HTTPParams> _http_params) {
   if (Started()) {
     throw std::runtime_error("HttpServer already started");
   }
@@ -108,6 +113,7 @@ void HttpServer::DoStart(const uint16_t _local_port,
   local_port = _local_port;
   local_url = StringUtil::Format("http://localhost:%d", local_port);
   remote_url = _remote_url;
+  http_params = std::move(_http_params);
   user_agent =
       StringUtil::Format("duckdb-ui/%s-%s(%s)", DuckDB::LibraryVersion(),
                          UI_EXTENSION_VERSION, DuckDB::Platform());
@@ -144,6 +150,7 @@ void HttpServer::DoStop() {
   }
 
   ddb_instance.reset();
+  http_params = nullptr;
   remote_url = "";
   local_port = 0;
 }
@@ -240,19 +247,47 @@ void HttpServer::HandleGetLocalToken(const httplib::Request &req,
   }
 }
 
+// Adapted from
+// https://github.com/duckdb/duckdb/blob/1f8b6839ea7864c3e3fb020574f67384cb58124c/src/main/http/http_util.cpp#L129-L147
+// Which is not currently exposed.
+void HttpServer::InitClientFromParams(httplib::Client &client) {
+  auto sec = static_cast<time_t>(http_params->timeout);
+  auto usec = static_cast<time_t>(http_params->timeout_usec);
+  client.set_keep_alive(true);
+  client.set_write_timeout(sec, usec);
+  client.set_read_timeout(sec, usec);
+  client.set_connection_timeout(sec, usec);
+
+  if (!http_params->http_proxy.empty()) {
+    client.set_proxy(http_params->http_proxy,
+                     static_cast<int>(http_params->http_proxy_port));
+
+    if (!http_params->http_proxy_username.empty()) {
+      client.set_proxy_basic_auth(http_params->http_proxy_username,
+                                  http_params->http_proxy_password);
+    }
+  }
+}
+
 void HttpServer::HandleGet(const httplib::Request &req,
                            httplib::Response &res) {
   // Create HTTP client to remote URL
   // TODO: Can this be created once and shared?
   httplib::Client client(remote_url);
-  client.set_keep_alive(true);
+  InitClientFromParams(client);
 
   if (IsEnvEnabled("ui_disable_server_certificate_verification")) {
     client.enable_server_certificate_verification(false);
   }
 
+  httplib::Headers headers = {{"User-Agent", user_agent}};
+  auto cookie = req.get_header_value("Cookie");
+  if (!cookie.empty()) {
+    headers.emplace("Cookie", cookie);
+  }
+
   // forward GET to remote URL
-  auto result = client.Get(req.path, req.params, {{"User-Agent", user_agent}});
+  auto result = client.Get(req.path, req.params, headers);
   if (!result) {
     res.status = 500;
     res.set_content("Could not fetch: '" + req.path + "' from '" + remote_url +
@@ -366,13 +401,63 @@ void HttpServer::DoHandleRun(const httplib::Request &req,
     });
   }
 
+  unique_ptr<SQLStatement> last_statement;
+
+  auto statements = connection->ExtractStatements(content);
+  auto statement_count = statements.size();
+
+  if (statement_count == 0) {
+    SetResponseErrorResult(res, "No statements");
+    return;
+  }
+
+  // If there's more than one statement, run all but the last.
+  if (statement_count > 1) {
+    for (auto i = 0; i < statement_count - 1; ++i) {
+      auto pending = connection->PendingQuery(std::move(statements[i]));
+      // Return any error found before execution.
+      if (pending->HasError()) {
+        SetResponseErrorResult(res, pending->GetError());
+        return;
+      }
+      // Execute tasks until result is ready (or there's an error).
+      auto exec_result = PendingExecutionResult::RESULT_NOT_READY;
+      while (!PendingQueryResult::IsResultReady(exec_result)) {
+        exec_result = pending->ExecuteTask();
+        if (exec_result == PendingExecutionResult::BLOCKED ||
+            exec_result == PendingExecutionResult::NO_TASKS_AVAILABLE) {
+          std::this_thread::sleep_for(std::chrono::milliseconds(1));
+        }
+      }
+      // Return any error found during execution.
+      switch (exec_result) {
+      case PendingExecutionResult::EXECUTION_ERROR:
+        SetResponseErrorResult(res, pending->GetError());
+        return;
+      case PendingExecutionResult::EXECUTION_FINISHED:
+      case PendingExecutionResult::RESULT_READY:
+        // ignore the result
+        pending->Execute();
+        break;
+      default:
+        SetResponseErrorResult(
+            res, StringUtil::Format("Unexpected PendingExecutionResult: %s",
+                                    exec_result));
+        return;
+      }
+    }
+  }
+
+  // Get the last statement.
+  auto &statement_to_run = statements[statement_count - 1];
+
   // We use a pending query so we can execute tasks and fetch chunks
   // incrementally. This enables cancellation.
   unique_ptr<PendingQueryResult> pending;
 
   // Create pending query, with request content as SQL.
   if (parameter_values.size() > 0) {
-    auto prepared = connection->Prepare(content);
+    auto prepared = connection->Prepare(std::move(statement_to_run));
     if (prepared->HasError()) {
       SetResponseErrorResult(res, prepared->GetError());
       return;
@@ -385,7 +470,7 @@ void HttpServer::DoHandleRun(const httplib::Request &req,
     }
     pending = prepared->PendingQuery(values, true);
   } else {
-    pending = connection->PendingQuery(content, true);
+    pending = connection->PendingQuery(std::move(statement_to_run), true);
   }
 
   if (pending->HasError()) {
@@ -433,7 +518,9 @@ void HttpServer::DoHandleRun(const httplib::Request &req,
     break;
   }
   default:
-    SetResponseErrorResult(res, "Unexpected PendingExecutionResult");
+    SetResponseErrorResult(
+        res, StringUtil::Format("Unexpected PendingExecutionResult: %s",
+                                exec_result));
     break;
   }
 }

src/include/http_server.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <duckdb.hpp>
+#include <duckdb/common/http_util.hpp>
 
 #define CPPHTTPLIB_OPENSSL_SUPPORT
 #include "httplib.hpp"
@@ -15,6 +16,7 @@
 namespace httplib = duckdb_httplib_openssl;
 
 namespace duckdb {
+class HTTPParams;
 class MemoryStream;
 
 namespace ui {
@@ -40,7 +42,8 @@ private:
   friend class Watcher;
 
   // Lifecycle
-  void DoStart(const uint16_t local_port, const std::string &remote_url);
+  void DoStart(const uint16_t local_port, const std::string &remote_url,
+               unique_ptr<HTTPParams>);
   void DoStop();
   void Run();
   void UpdateDatabaseInstance(shared_ptr<DatabaseInstance> context_db);
@@ -67,6 +70,7 @@ private:
 
   // Misc
   shared_ptr<DatabaseInstance> LockDatabaseInstance();
+  void InitClientFromParams(httplib::Client &);
 
   uint16_t local_port;
   std::string local_url;
@@ -77,6 +81,7 @@ private:
   unique_ptr<std::thread> main_thread;
   unique_ptr<EventDispatcher> event_dispatcher;
   unique_ptr<Watcher> watcher;
+  unique_ptr<HTTPParams> http_params;
 
   static unique_ptr<HttpServer> server_instance;
 };

src/include/ui_extension.hpp

@@ -6,7 +6,12 @@ namespace duckdb {
 
 class UiExtension : public Extension {
 public:
+#ifdef DUCKDB_CPP_EXTENSION_ENTRY
+  void Load(ExtensionLoader &loader) override;
+#else
   void Load(DuckDB &db) override;
+#endif
+
   std::string Name() override;
   std::string Version() const override;
 };

src/include/utils/helpers.hpp

@@ -1,7 +1,9 @@
 #pragma once
 
 #include <duckdb.hpp>
+#ifndef DUCKDB_CPP_EXTENSION_ENTRY
 #include <duckdb/main/extension_util.hpp>
+#endif
 #include <type_traits>
 
 namespace duckdb {
@@ -64,6 +66,15 @@ void TableFunc(ClientContext &context, TableFunctionInput &input,
   output.SetValue(0, 0, result);
 }
 
+#ifdef DUCKDB_CPP_EXTENSION_ENTRY
+template <typename Func, Func func>
+void RegisterTF(ExtensionLoader &loader, const char *name) {
+  TableFunction tf(name, {}, internal::TableFunc<Func, func>,
+                   internal::SingleStringResultBind,
+                   RunOnceTableFunctionState::Init);
+  loader.RegisterFunction(tf);
+}
+#else
 template <typename Func, Func func>
 void RegisterTF(DatabaseInstance &instance, const char *name) {
   TableFunction tf(name, {}, internal::TableFunc<Func, func>,
@@ -71,10 +82,16 @@ void RegisterTF(DatabaseInstance &instance, const char *name) {
                    RunOnceTableFunctionState::Init);
   ExtensionUtil::RegisterFunction(instance, tf);
 }
+#endif
 
 } // namespace internal
 
+#ifdef DUCKDB_CPP_EXTENSION_ENTRY
+#define REGISTER_TF(name, func)                                                \
+  internal::RegisterTF<decltype(&func), &func>(loader, name)
+#else
 #define REGISTER_TF(name, func)                                                \
   internal::RegisterTF<decltype(&func), &func>(instance, name)
+#endif
 
 } // namespace duckdb

src/ui_extension.cpp

@@ -13,6 +13,7 @@
 
 #ifdef _WIN32
 #define OPEN_COMMAND "start"
+#undef CreateDirectory // avoid being transformed to `CreateDirectoryA`
 #elif __linux__
 #define OPEN_COMMAND "xdg-open"
 #else
@@ -81,7 +82,12 @@ void InitStorageExtension(duckdb::DatabaseInstance &db) {
   config.storage_extensions[STORAGE_EXTENSION_KEY] = std::move(ext);
 }
 
+#ifdef DUCKDB_CPP_EXTENSION_ENTRY
+static void LoadInternal(ExtensionLoader &loader) {
+  auto &instance = loader.GetDatabaseInstance();
+#else
 static void LoadInternal(DatabaseInstance &instance) {
+#endif
   InitStorageExtension(instance);
 
   // If the server is already running we need to update the database instance
@@ -128,11 +134,20 @@ static void LoadInternal(DatabaseInstance &instance) {
     TableFunction tf("ui_is_started", {}, IsUIStartedTableFunc,
                      internal::SingleBoolResultBind,
                      RunOnceTableFunctionState::Init);
+#ifdef DUCKDB_CPP_EXTENSION_ENTRY
+    loader.RegisterFunction(tf);
+#else
     ExtensionUtil::RegisterFunction(instance, tf);
+#endif
   }
 }
 
+#ifdef DUCKDB_CPP_EXTENSION_ENTRY
+void UiExtension::Load(ExtensionLoader &loader) { LoadInternal(loader); }
+#else
 void UiExtension::Load(DuckDB &db) { LoadInternal(*db.instance); }
+#endif
+
 std::string UiExtension::Name() { return "ui"; }
 
 std::string UiExtension::Version() const { return UI_EXTENSION_VERSION; }
@@ -141,10 +156,14 @@ std::string UiExtension::Version() const { return UI_EXTENSION_VERSION; }
 
 extern "C" {
 
+#ifdef DUCKDB_CPP_EXTENSION_ENTRY
+DUCKDB_CPP_EXTENSION_ENTRY(ui, loader) { duckdb::LoadInternal(loader); }
+#else
 DUCKDB_EXTENSION_API void ui_init(duckdb::DatabaseInstance &db) {
   duckdb::DuckDB db_wrapper(db);
   db_wrapper.LoadExtension<duckdb::UiExtension>();
 }
+#endif
 
 DUCKDB_EXTENSION_API const char *ui_version() {
   return duckdb::DuckDB::LibraryVersion();