Initial commit

scripts/bootstrap-template.py

@@ -0,0 +1,199 @@
+#!/usr/bin/python3
+
+import os
+import re
+import shutil
+import sys
+from pathlib import Path
+
+
+def is_snake_case(s: str) -> bool:
+    """
+    Check if the provided string is in snake_case format.
+    Snake case is lower case with words separated by underscores, and it can contain digits.
+
+    Args:
+        s (str): String to check.
+
+    Returns:
+        bool: True if the string is in snake_case, False otherwise.
+    """
+    pattern = r"^[a-z0-9]+(_[a-z0-9]+)*$"
+    return bool(re.match(pattern, s))
+
+
+def to_camel_case(snake_str: str) -> str:
+    """
+    Convert a snake_case string to camelCase.
+
+    Args:
+        snake_str (str): String in snake_case to convert.
+
+    Returns:
+        str: Converted string in camelCase.
+    """
+    return "".join(x.capitalize() for x in snake_str.lower().split("_"))
+
+
+def replace(file_name: str, to_find: str, to_replace: str) -> None:
+    """
+    Replace occurrences of a string within a file, ensuring placeholders are handled.
+    The function replaces the `to_find` string with `to_replace`, adds a placeholder,
+    and skips lines with placeholders already in place.
+
+    Args:
+        file_name (str): Path to the file to perform replacement in.
+        to_find (str): String to search for in the file.
+        to_replace (str): String to replace `to_find` with.
+
+    Returns:
+        None
+    """
+    with open(file_name, "r", encoding="utf8") as file:
+        filedata = file.readlines()
+
+    new_filedata = []
+    for line in filedata:
+        # Skip lines that have already been replaced by checking for placeholder
+        if "__REPLACEMENT_DONE__" in line:
+            new_filedata.append(line)
+            continue
+
+        modified_line = line.replace(
+            to_find,
+            to_replace,
+        )
+        modified_line = modified_line.replace(
+            to_find.capitalize(), to_camel_case(to_replace)
+        )
+        modified_line = modified_line.replace(
+            to_find.upper(),
+            to_replace.upper(),
+        )
+
+        # Add placeholder once after all replacements
+        if to_find in line or to_find.capitalize() in line or to_find.upper() in line:
+            modified_line += "__REPLACEMENT_DONE__"
+
+        new_filedata.append(modified_line)
+
+    with open(file_name, "w", encoding="utf8") as file:
+        file.writelines(new_filedata)
+
+
+def replace_everywhere(to_find: str, to_replace: str) -> None:
+    """
+    Replace a string in all files in the project.
+
+    Args:
+        to_find (str): String to search for in the file.
+        to_replace (str): String to replace `to_find` with.
+
+    Returns:
+        None
+    """
+    for path in files_to_search:
+        replace(path, to_find, to_replace)
+        replace(path, to_find.capitalize(), to_camel_case(to_replace))
+        replace(path, to_find.upper(), to_replace.upper())
+
+    replace("./CMakeLists.txt", to_find, to_replace)
+    replace("./Makefile", to_find, to_replace)
+    replace("./Makefile", to_find.capitalize(), to_camel_case(to_replace))
+    replace("./Makefile", to_find.upper(), to_replace.upper())
+    replace("./README.md", to_find, to_replace)
+    replace("./extension_config.cmake", to_find, to_replace)
+    replace("./scripts/setup-custom-toolchain.sh", to_find, to_replace)
+
+
+def remove_placeholder() -> None:
+    """
+    Remove the placeholder from all files.
+
+    Returns:
+        None
+    """
+    for path in files_to_search:
+        replace_placeholders(path)
+
+    replace_placeholders("./CMakeLists.txt")
+    replace_placeholders("./Makefile")
+    replace_placeholders("./Makefile")
+    replace_placeholders("./Makefile")
+    replace_placeholders("./README.md")
+    replace_placeholders("./extension_config.cmake")
+    replace_placeholders("./scripts/setup-custom-toolchain.sh")
+
+
+def replace_placeholders(file_name: str) -> None:
+    """
+    Remove the placeholder from a file.
+
+    Args:
+        file_name (str): Path to the file to remove the placeholder from.
+
+    Returns:
+        None
+    """
+    with open(file_name, "r", encoding="utf8") as file:
+        filedata = file.read()
+
+    # Remove all placeholders
+    filedata = filedata.replace("__REPLACEMENT_DONE__", "")
+
+    with open(file_name, "w", encoding="utf8") as file:
+        file.write(filedata)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        raise Exception(
+            "usage: python3 bootstrap-template.py <name_for_extension_in_snake_case>"
+        )
+
+    name_extension = sys.argv[1]
+
+    if name_extension[0].isdigit():
+        raise Exception("Please dont start your extension name with a number.")
+
+    if not is_snake_case(name_extension):
+        raise Exception(
+            "Please enter the name of your extension in valid snake_case containing only lower case letters and numbers"
+        )
+
+    shutil.copyfile("docs/NEXT_README.md", "README.md")
+    os.remove("docs/NEXT_README.md")
+    os.remove("docs/README.md")
+
+    files_to_search = []
+    files_to_search.extend(Path("./.github").rglob("./**/*.yml"))
+    files_to_search.extend(Path("./test").rglob("./**/*.test"))
+    files_to_search.extend(Path("./src").rglob("./**/*.hpp"))
+    files_to_search.extend(Path("./src").rglob("./**/*.cpp"))
+    files_to_search.extend(Path("./src").rglob("./**/*.txt"))
+    files_to_search.extend(Path("./src").rglob("./*.md"))
+
+    replace_everywhere("quack", name_extension)
+    replace_everywhere("Quack", name_extension.capitalize())
+    replace_everywhere("<extension_name>", name_extension)
+
+    remove_placeholder()
+
+    string_to_replace = name_extension
+    string_to_find = "quack"
+
+    # rename files
+    os.rename(f"test/sql/{string_to_find}.test", f"test/sql/{string_to_replace}.test")
+    os.rename(
+        f"src/{string_to_find}_extension.cpp", f"src/{string_to_replace}_extension.cpp"
+    )
+    os.rename(
+        f"src/include/{string_to_find}_extension.hpp",
+        f"src/include/{string_to_replace}_extension.hpp",
+    )
+
+    # remove template-specific files
+    os.remove(".github/workflows/ExtensionTemplate.yml")
+
+    # finally, remove this bootstrap file
+    os.remove(__file__)

scripts/extension-upload.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Extension upload script
+
+# Usage: ./extension-upload.sh <name> <extension_version> <duckdb_version> <architecture> <s3_bucket> <copy_to_latest> <copy_to_versioned>
+# <name>                : Name of the extension
+# <extension_version>   : Version (commit / version tag) of the extension
+# <duckdb_version>      : Version (commit / version tag) of DuckDB
+# <architecture>        : Architecture target of the extension binary
+# <s3_bucket>           : S3 bucket to upload to
+# <copy_to_latest>      : Set this as the latest version ("true" / "false", default: "false")
+# <copy_to_versioned>   : Set this as a versioned version that will prevent its deletion
+
+set -e
+
+if [[ $4 == wasm* ]]; then
+  ext="/tmp/extension/$1.duckdb_extension.wasm"
+else
+  ext="/tmp/extension/$1.duckdb_extension"
+fi
+
+echo $ext
+
+script_dir="$(dirname "$(readlink -f "$0")")"
+
+# calculate SHA256 hash of extension binary
+cat $ext > $ext.append
+
+if [[ $4 == wasm* ]]; then
+  # 0 for custom section
+  # 113 in hex = 275 in decimal, total lenght of what follows (1 + 16 + 2 + 256)
+  # [1(continuation) + 0010011(payload) = \x93, 0(continuation) + 10(payload) = \x02]
+  echo -n -e '\x00' >> $ext.append
+  echo -n -e '\x93\x02' >> $ext.append
+  # 10 in hex = 16 in decimal, lenght of name, 1 byte
+  echo -n -e '\x10' >> $ext.append
+  echo -n -e 'duckdb_signature' >> $ext.append
+  # the name of the WebAssembly custom section, 16 bytes
+  # 100 in hex, 256 in decimal
+  # [1(continuation) + 0000000(payload) = ff, 0(continuation) + 10(payload)],
+  # for a grand total of 2 bytes
+  echo -n -e '\x80\x02' >> $ext.append
+fi
+
+# (Optionally) Sign binary
+if [ "$DUCKDB_EXTENSION_SIGNING_PK" != "" ]; then
+  echo "$DUCKDB_EXTENSION_SIGNING_PK" > private.pem
+  $script_dir/../duckdb/scripts/compute-extension-hash.sh $ext.append > $ext.hash
+  openssl pkeyutl -sign -in $ext.hash -inkey private.pem -pkeyopt digest:sha256 -out $ext.sign
+  rm -f private.pem
+fi
+
+# Signature is always there, potentially defaulting to 256 zeros
+truncate -s 256 $ext.sign
+
+# append signature to extension binary
+cat $ext.sign >> $ext.append
+
+# compress extension binary
+if [[ $4 == wasm_* ]]; then
+  brotli < $ext.append > "$ext.compressed"
+else
+  gzip < $ext.append > "$ext.compressed"
+fi
+
+set -e
+
+# Abort if AWS key is not set
+if [ -z "$AWS_ACCESS_KEY_ID" ]; then
+    echo "No AWS key found, skipping.."
+    exit 0
+fi
+
+# upload versioned version
+if [[ $7 = 'true' ]]; then
+  if [[ $4 == wasm* ]]; then
+    aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm"
+  else
+    aws s3 cp $ext.compressed s3://$5/$1/$2/$3/$4/$1.duckdb_extension.gz --acl public-read
+  fi
+fi
+
+# upload to latest version
+if [[ $6 = 'true' ]]; then
+  if [[ $4 == wasm* ]]; then
+    aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.wasm --acl public-read --content-encoding br --content-type="application/wasm"
+  else
+    aws s3 cp $ext.compressed s3://$5/$3/$4/$1.duckdb_extension.gz --acl public-read
+  fi
+fi

scripts/setup-custom-toolchain.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# This is an example script that can be used to install additional toolchain dependencies. Feel free to remove this script
+# if no additional toolchains are required
+
+# To enable this script, set the `custom_toolchain_script` option to true when calling the reusable workflow
+# `.github/workflows/_extension_distribution.yml` from `https://github.com/duckdb/extension-ci-tools`
+
+# note that the $DUCKDB_PLATFORM environment variable can be used to discern between the platforms
+echo "This is the sample custom toolchain script running for architecture '$DUCKDB_PLATFORM' for the quack extension."
+