add duckdb-ui-client & other ts pkgs (#10)

* add duckdb-ui-client & other ts pkgs

* workflow fixes

* fix working dir

* no sparse checkout; specify package.json path

* path to pnpm-lock.yaml

* add check & build test

* workflow step descriptions

* use comments & names

* one more naming tweak
This commit is contained in:
Jeff Raymakers
2025-06-13 09:06:55 -07:00
parent d6cc9eeea4
commit 0edb52054a
133 changed files with 11112 additions and 4 deletions

View File

@@ -0,0 +1,58 @@
// DuckDB's physical storage and binary serialization format is little endian.
const littleEndian = true;
export function getInt8(dataView: DataView, offset: number): number {
return dataView.getInt8(offset);
}
export function getUInt8(dataView: DataView, offset: number): number {
return dataView.getUint8(offset);
}
export function getInt16(dataView: DataView, offset: number): number {
return dataView.getInt16(offset, littleEndian);
}
export function getUInt16(dataView: DataView, offset: number): number {
return dataView.getUint16(offset, littleEndian);
}
export function getInt32(dataView: DataView, offset: number): number {
return dataView.getInt32(offset, littleEndian);
}
export function getUInt32(dataView: DataView, offset: number): number {
return dataView.getUint32(offset, littleEndian);
}
export function getInt64(dataView: DataView, offset: number): bigint {
return dataView.getBigInt64(offset, littleEndian);
}
export function getUInt64(dataView: DataView, offset: number): bigint {
return dataView.getBigUint64(offset, littleEndian);
}
export function getFloat32(dataView: DataView, offset: number): number {
return dataView.getFloat32(offset, littleEndian);
}
export function getFloat64(dataView: DataView, offset: number): number {
return dataView.getFloat64(offset, littleEndian);
}
export function getInt128(dataView: DataView, offset: number): bigint {
const lower = getUInt64(dataView, offset);
const upper = getInt64(dataView, offset + 8);
return (upper << BigInt(64)) + lower;
}
export function getUInt128(dataView: DataView, offset: number): bigint {
const lower = getUInt64(dataView, offset);
const upper = getUInt64(dataView, offset + 8);
return (BigInt.asUintN(64, upper) << BigInt(64)) | BigInt.asUintN(64, lower);
}
export function getBoolean(dataView: DataView, offset: number): boolean {
return getUInt8(dataView, offset) !== 0;
}

View File

@@ -0,0 +1,178 @@
import {
ARRAY,
DECIMAL,
DuckDBBigIntType,
DuckDBBitType,
DuckDBBlobType,
DuckDBBooleanType,
DuckDBDateType,
DuckDBDoubleType,
DuckDBFloatType,
DuckDBHugeIntType,
DuckDBIntegerType,
DuckDBIntervalType,
DuckDBSmallIntType,
DuckDBTimestampMillisecondsType,
DuckDBTimestampNanosecondsType,
DuckDBTimestampSecondsType,
DuckDBTimestampType,
DuckDBTimestampTZType,
DuckDBTimeType,
DuckDBTimeTZType,
DuckDBTinyIntType,
DuckDBType,
DuckDBUBigIntType,
DuckDBUHugeIntType,
DuckDBUIntegerType,
DuckDBUSmallIntType,
DuckDBUTinyIntType,
DuckDBUUIDType,
DuckDBVarCharType,
DuckDBVarIntType,
ENUM,
JSONType,
LIST,
MAP,
STRUCT,
UNION,
} from '@duckdb/data-types';
import { LogicalTypeId } from '../../serialization/constants/LogicalTypeId.js';
import { TypeIdAndInfo } from '../../serialization/types/TypeInfo.js';
import {
getArrayTypeInfo,
getDecimalTypeInfo,
getEnumTypeInfo,
getListTypeInfo,
getMapTypeInfos,
getStructTypeInfo,
} from './typeInfoGetters.js';
/** Return the DuckDBType corresponding to the given TypeIdAndInfo. */
export function duckDBTypeFromTypeIdAndInfo(
typeIdAndInfo: TypeIdAndInfo,
): DuckDBType {
const { id, typeInfo } = typeIdAndInfo;
const alias = typeInfo?.alias;
switch (id) {
case LogicalTypeId.BOOLEAN:
return DuckDBBooleanType.create(alias);
case LogicalTypeId.TINYINT:
return DuckDBTinyIntType.create(alias);
case LogicalTypeId.SMALLINT:
return DuckDBSmallIntType.create(alias);
case LogicalTypeId.INTEGER:
return DuckDBIntegerType.create(alias);
case LogicalTypeId.BIGINT:
return DuckDBBigIntType.create(alias);
case LogicalTypeId.DATE:
return DuckDBDateType.create(alias);
case LogicalTypeId.TIME:
return DuckDBTimeType.create(alias);
case LogicalTypeId.TIMESTAMP_SEC:
return DuckDBTimestampSecondsType.create(alias);
case LogicalTypeId.TIMESTAMP_MS:
return DuckDBTimestampMillisecondsType.create(alias);
case LogicalTypeId.TIMESTAMP:
return DuckDBTimestampType.create(alias);
case LogicalTypeId.TIMESTAMP_NS:
return DuckDBTimestampNanosecondsType.create(alias);
case LogicalTypeId.DECIMAL: {
const { width, scale } = getDecimalTypeInfo(typeInfo);
return DECIMAL(width, scale, alias);
}
case LogicalTypeId.FLOAT:
return DuckDBFloatType.create(alias);
case LogicalTypeId.DOUBLE:
return DuckDBDoubleType.create(alias);
case LogicalTypeId.CHAR:
case LogicalTypeId.VARCHAR:
// Minor optimization for JSON type to avoid creating new type object.
if (alias === JSONType.alias) {
return JSONType;
}
return DuckDBVarCharType.create(alias);
case LogicalTypeId.BLOB:
return DuckDBBlobType.create(alias);
case LogicalTypeId.INTERVAL:
return DuckDBIntervalType.create(alias);
case LogicalTypeId.UTINYINT:
return DuckDBUTinyIntType.create(alias);
case LogicalTypeId.USMALLINT:
return DuckDBUSmallIntType.create(alias);
case LogicalTypeId.UINTEGER:
return DuckDBUIntegerType.create(alias);
case LogicalTypeId.UBIGINT:
return DuckDBUBigIntType.create(alias);
case LogicalTypeId.TIMESTAMP_TZ:
return DuckDBTimestampTZType.create(alias);
case LogicalTypeId.TIME_TZ:
return DuckDBTimeTZType.create(alias);
case LogicalTypeId.BIT:
return DuckDBBitType.create(alias);
case LogicalTypeId.VARINT:
return DuckDBVarIntType.create(alias);
case LogicalTypeId.UHUGEINT:
return DuckDBUHugeIntType.create(alias);
case LogicalTypeId.HUGEINT:
return DuckDBHugeIntType.create(alias);
case LogicalTypeId.UUID:
return DuckDBUUIDType.create(alias);
case LogicalTypeId.STRUCT: {
const { childTypes } = getStructTypeInfo(typeInfo);
const entries: Record<string, DuckDBType> = {};
for (const [key, valueTypeIdAndInfo] of childTypes) {
entries[key] = duckDBTypeFromTypeIdAndInfo(valueTypeIdAndInfo);
}
return STRUCT(entries, alias);
}
case LogicalTypeId.LIST: {
const { childType } = getListTypeInfo(typeInfo);
return LIST(duckDBTypeFromTypeIdAndInfo(childType), alias);
}
case LogicalTypeId.MAP: {
const { keyType, valueType } = getMapTypeInfos(typeInfo);
return MAP(
duckDBTypeFromTypeIdAndInfo(keyType),
duckDBTypeFromTypeIdAndInfo(valueType),
alias,
);
}
case LogicalTypeId.ENUM: {
const { values } = getEnumTypeInfo(typeInfo);
return ENUM(values, alias);
}
case LogicalTypeId.UNION: {
const { childTypes } = getStructTypeInfo(typeInfo);
const members: Record<string, DuckDBType> = {};
for (const [key, valueTypeIdAndInfo] of childTypes) {
members[key] = duckDBTypeFromTypeIdAndInfo(valueTypeIdAndInfo);
}
return UNION(members, alias);
}
case LogicalTypeId.ARRAY: {
const { childType, size } = getArrayTypeInfo(typeInfo);
return ARRAY(duckDBTypeFromTypeIdAndInfo(childType), size, alias);
}
default:
throw new Error(`type id not implemented: ${typeIdAndInfo.id}`);
}
}

View File

@@ -0,0 +1,271 @@
import {
DuckDBArrayValue,
DuckDBBitValue,
DuckDBBlobValue,
DuckDBDateValue,
DuckDBDecimalValue,
DuckDBIntervalValue,
DuckDBListValue,
DuckDBMapValue,
DuckDBStructValue,
DuckDBTimeTZValue,
DuckDBTimeValue,
DuckDBTimestampMicrosecondsValue,
DuckDBTimestampMillisecondsValue,
DuckDBTimestampNanosecondsValue,
DuckDBTimestampSecondsValue,
DuckDBTimestampTZValue,
DuckDBUUIDValue,
DuckDBValue,
getVarIntFromBytes,
} from '@duckdb/data-values';
import { LogicalTypeId } from '../../serialization/constants/LogicalTypeId.js';
import { TypeIdAndInfo } from '../../serialization/types/TypeInfo.js';
import { Vector } from '../../serialization/types/Vector.js';
import {
getBoolean,
getFloat32,
getFloat64,
getInt128,
getInt16,
getInt32,
getInt64,
getInt8,
getUInt128,
getUInt16,
getUInt32,
getUInt64,
getUInt8,
} from './dataViewReaders.js';
import { isRowValid } from './isRowValid.js';
import {
getArrayTypeInfo,
getDecimalTypeInfo,
getEnumTypeInfo,
getListTypeInfo,
getMapTypeInfos,
getStructTypeInfo,
} from './typeInfoGetters.js';
import {
getArrayVector,
getDataListVector,
getDataVector,
getListVector,
getStringVector,
getVectorListVector,
} from './vectorGetters.js';
/** Return the DuckDBValue at the given index in the given Vector with the type described by the given TypeIdAndInfo. */
export function duckDBValueFromVector(
typeIdAndInfo: TypeIdAndInfo,
vector: Vector,
rowIndex: number,
): DuckDBValue {
if (!isRowValid(vector.validity, rowIndex)) return null;
const { id, typeInfo } = typeIdAndInfo;
switch (id) {
case LogicalTypeId.BOOLEAN:
return getBoolean(getDataVector(vector).data, rowIndex);
case LogicalTypeId.TINYINT:
return getInt8(getDataVector(vector).data, rowIndex);
case LogicalTypeId.SMALLINT:
return getInt16(getDataVector(vector).data, rowIndex * 2);
case LogicalTypeId.INTEGER:
return getInt32(getDataVector(vector).data, rowIndex * 4);
case LogicalTypeId.BIGINT:
return getInt64(getDataVector(vector).data, rowIndex * 8);
case LogicalTypeId.DATE:
return new DuckDBDateValue(
getInt32(getDataVector(vector).data, rowIndex * 4),
);
case LogicalTypeId.TIME:
return new DuckDBTimeValue(
getInt64(getDataVector(vector).data, rowIndex * 8),
);
case LogicalTypeId.TIMESTAMP_SEC:
return new DuckDBTimestampSecondsValue(
getInt64(getDataVector(vector).data, rowIndex * 8),
);
case LogicalTypeId.TIMESTAMP_MS:
return new DuckDBTimestampMillisecondsValue(
getInt64(getDataVector(vector).data, rowIndex * 8),
);
case LogicalTypeId.TIMESTAMP:
return new DuckDBTimestampMicrosecondsValue(
getInt64(getDataVector(vector).data, rowIndex * 8),
);
case LogicalTypeId.TIMESTAMP_NS:
return new DuckDBTimestampNanosecondsValue(
getInt64(getDataVector(vector).data, rowIndex * 8),
);
case LogicalTypeId.DECIMAL: {
const { width, scale } = getDecimalTypeInfo(typeInfo);
if (width <= 4) {
return new DuckDBDecimalValue(
BigInt(getInt16(getDataVector(vector).data, rowIndex * 2)),
scale,
);
} else if (width <= 9) {
return new DuckDBDecimalValue(
BigInt(getInt32(getDataVector(vector).data, rowIndex * 4)),
scale,
);
} else if (width <= 18) {
return new DuckDBDecimalValue(
getInt64(getDataVector(vector).data, rowIndex * 8),
scale,
);
} else if (width <= 38) {
return new DuckDBDecimalValue(
getInt128(getDataVector(vector).data, rowIndex * 16),
scale,
);
}
throw new Error(`unsupported decimal width: ${width}`);
}
case LogicalTypeId.FLOAT:
return getFloat32(getDataVector(vector).data, rowIndex * 4);
case LogicalTypeId.DOUBLE:
return getFloat64(getDataVector(vector).data, rowIndex * 8);
case LogicalTypeId.CHAR:
case LogicalTypeId.VARCHAR:
return getStringVector(vector).data[rowIndex];
case LogicalTypeId.BLOB: {
const dv = getDataListVector(vector).data[rowIndex];
return new DuckDBBlobValue(
new Uint8Array(dv.buffer, dv.byteOffset, dv.byteLength),
);
}
case LogicalTypeId.INTERVAL: {
const { data } = getDataVector(vector);
const months = getInt32(data, rowIndex * 16 + 0);
const days = getInt32(data, rowIndex * 16 + 4);
const micros = getInt64(data, rowIndex * 16 + 8);
return new DuckDBIntervalValue(months, days, micros);
}
case LogicalTypeId.UTINYINT:
return getUInt8(getDataVector(vector).data, rowIndex);
case LogicalTypeId.USMALLINT:
return getUInt16(getDataVector(vector).data, rowIndex * 2);
case LogicalTypeId.UINTEGER:
return getUInt32(getDataVector(vector).data, rowIndex * 4);
case LogicalTypeId.UBIGINT:
return getUInt64(getDataVector(vector).data, rowIndex * 8);
case LogicalTypeId.TIMESTAMP_TZ:
return new DuckDBTimestampTZValue(
getInt64(getDataVector(vector).data, rowIndex * 8),
);
case LogicalTypeId.TIME_TZ:
return DuckDBTimeTZValue.fromBits(
getUInt64(getDataVector(vector).data, rowIndex * 8),
);
case LogicalTypeId.BIT: {
const dv = getDataListVector(vector).data[rowIndex];
return new DuckDBBitValue(
new Uint8Array(dv.buffer, dv.byteOffset, dv.byteLength),
);
}
case LogicalTypeId.VARINT: {
const dv = getDataListVector(vector).data[rowIndex];
return getVarIntFromBytes(
new Uint8Array(dv.buffer, dv.byteOffset, dv.byteLength),
);
}
case LogicalTypeId.UHUGEINT:
return getUInt128(getDataVector(vector).data, rowIndex * 16);
case LogicalTypeId.HUGEINT:
return getInt128(getDataVector(vector).data, rowIndex * 16);
case LogicalTypeId.UUID:
return DuckDBUUIDValue.fromStoredHugeint(
getInt128(getDataVector(vector).data, rowIndex * 16),
);
case LogicalTypeId.STRUCT: {
const { childTypes } = getStructTypeInfo(typeInfo);
const { data } = getVectorListVector(vector);
return new DuckDBStructValue(
Array.from({ length: childTypes.length }).map((_, i) => ({
key: childTypes[i][0],
value: duckDBValueFromVector(childTypes[i][1], data[i], rowIndex),
})),
);
}
case LogicalTypeId.LIST: {
const { childType } = getListTypeInfo(typeInfo);
const { child, entries } = getListVector(vector);
const { offset, length } = entries[rowIndex];
return new DuckDBListValue(
Array.from({ length }).map((_, i) =>
duckDBValueFromVector(childType, child, offset + i),
),
);
}
case LogicalTypeId.MAP: {
const { keyType, valueType } = getMapTypeInfos(typeInfo);
const { child, entries } = getListVector(vector);
const { offset, length } = entries[rowIndex];
const { data } = getVectorListVector(child);
return new DuckDBMapValue(
Array.from({ length }).map((_, i) => ({
key: duckDBValueFromVector(keyType, data[0], offset + i),
value: duckDBValueFromVector(valueType, data[1], offset + i),
})),
);
}
case LogicalTypeId.ENUM: {
const { values } = getEnumTypeInfo(typeInfo);
if (values.length < 256) {
return values[getUInt8(getDataVector(vector).data, rowIndex)];
} else if (values.length < 65536) {
return values[getUInt16(getDataVector(vector).data, rowIndex * 2)];
} else if (values.length < 4294967296) {
return values[getUInt32(getDataVector(vector).data, rowIndex * 4)];
}
throw new Error(`unsupported enum size: values.length=${values.length}`);
}
case LogicalTypeId.UNION: {
const { childTypes } = getStructTypeInfo(typeInfo);
const { data } = getVectorListVector(vector);
const tag = Number(
duckDBValueFromVector(childTypes[0][1], data[0], rowIndex),
);
const altIndex = tag + 1;
return duckDBValueFromVector(
childTypes[altIndex][1],
data[altIndex],
rowIndex,
);
}
case LogicalTypeId.ARRAY: {
const { childType, size } = getArrayTypeInfo(typeInfo);
const { child } = getArrayVector(vector);
return new DuckDBArrayValue(
Array.from({ length: size }).map((_, i) =>
duckDBValueFromVector(childType, child, rowIndex * size + i),
),
);
}
default:
throw new Error(`type not implemented: ${id}`);
}
}

View File

@@ -0,0 +1,7 @@
import { getUInt64 } from './dataViewReaders.js';
export function isRowValid(validity: DataView | null, row: number): boolean {
if (!validity) return true;
const bigint = getUInt64(validity, Math.floor(row / 64) * 8);
return (bigint & (1n << BigInt(row % 64))) !== 0n;
}

View File

@@ -0,0 +1,93 @@
import {
ArrayTypeInfo,
DecimalTypeInfo,
EnumTypeInfo,
ListTypeInfo,
StructTypeInfo,
TypeIdAndInfo,
TypeInfo,
} from '../../serialization/types/TypeInfo.js';
export function getArrayTypeInfo(
typeInfo: TypeInfo | undefined,
): ArrayTypeInfo {
if (!typeInfo) {
throw new Error(`ARRAY has no typeInfo!`);
}
if (typeInfo.kind !== 'array') {
throw new Error(`ARRAY has unexpected typeInfo.kind: ${typeInfo.kind}`);
}
return typeInfo;
}
export function getDecimalTypeInfo(
typeInfo: TypeInfo | undefined,
): DecimalTypeInfo {
if (!typeInfo) {
throw new Error(`DECIMAL has no typeInfo!`);
}
if (typeInfo.kind !== 'decimal') {
throw new Error(`DECIMAL has unexpected typeInfo.kind: ${typeInfo.kind}`);
}
return typeInfo;
}
export function getEnumTypeInfo(typeInfo: TypeInfo | undefined): EnumTypeInfo {
if (!typeInfo) {
throw new Error(`ENUM has no typeInfo!`);
}
if (typeInfo.kind !== 'enum') {
throw new Error(`ENUM has unexpected typeInfo.kind: ${typeInfo.kind}`);
}
return typeInfo;
}
export function getListTypeInfo(typeInfo: TypeInfo | undefined): ListTypeInfo {
if (!typeInfo) {
throw new Error(`LIST has no typeInfo!`);
}
if (typeInfo.kind !== 'list') {
throw new Error(`LIST has unexpected typeInfo.kind: ${typeInfo.kind}`);
}
return typeInfo;
}
export function getStructTypeInfo(
typeInfo: TypeInfo | undefined,
): StructTypeInfo {
if (!typeInfo) {
throw new Error(`STRUCT has no typeInfo!`);
}
if (typeInfo.kind !== 'struct') {
throw new Error(`STRUCT has unexpected typeInfo.kind: ${typeInfo.kind}`);
}
return typeInfo;
}
export function getMapTypeInfos(typeInfo: TypeInfo | undefined): {
keyType: TypeIdAndInfo;
valueType: TypeIdAndInfo;
} {
// MAP = LIST(STRUCT(key KEY_TYPE, value VALUE_TYPE))
const { childType } = getListTypeInfo(typeInfo);
const { childTypes } = getStructTypeInfo(childType.typeInfo);
if (childTypes.length !== 2) {
throw new Error(
`MAP childType has unexpected childTypes length: ${childTypes.length}`,
);
}
if (childTypes[0].length !== 2) {
throw new Error(
`MAP childType has unexpected childTypes[0] length: ${childTypes[0].length}`,
);
}
if (childTypes[1].length !== 2) {
throw new Error(
`MAP childType has unexpected childTypes[1] length: ${childTypes[1].length}`,
);
}
return {
keyType: childTypes[0][1],
valueType: childTypes[1][1],
};
}

View File

@@ -0,0 +1,51 @@
import {
ArrayVector,
DataListVector,
DataVector,
ListVector,
StringVector,
Vector,
VectorListVector,
} from '../../serialization/types/Vector.js';
export function getDataVector(vector: Vector): DataVector {
if (vector.kind !== 'data') {
throw new Error(`Unexpected vector.kind: ${vector.kind}`);
}
return vector;
}
export function getStringVector(vector: Vector): StringVector {
if (vector.kind !== 'string') {
throw new Error(`Unexpected vector.kind: ${vector.kind}`);
}
return vector;
}
export function getDataListVector(vector: Vector): DataListVector {
if (vector.kind !== 'datalist') {
throw new Error(`Unexpected vector.kind: ${vector.kind}`);
}
return vector;
}
export function getVectorListVector(vector: Vector): VectorListVector {
if (vector.kind !== 'vectorlist') {
throw new Error(`Unexpected vector.kind: ${vector.kind}`);
}
return vector;
}
export function getListVector(vector: Vector): ListVector {
if (vector.kind !== 'list') {
throw new Error(`Unexpected vector.kind: ${vector.kind}`);
}
return vector;
}
export function getArrayVector(vector: Vector): ArrayVector {
if (vector.kind !== 'array') {
throw new Error(`Unexpected vector.kind: ${vector.kind}`);
}
return vector;
}