Merge branch 'duckdb:main' into main

src/duckdb/src/common/extra_type_info.cpp

@@ -10,21 +10,12 @@
 namespace duckdb {
 
 //===--------------------------------------------------------------------===//
-// Extra Type Info
+// Extension Type Info
 //===--------------------------------------------------------------------===//
-ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type) : type(type) {
-}
-ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type, string alias) : type(type), alias(std::move(alias)) {
-}
-ExtraTypeInfo::~ExtraTypeInfo() {
-}
-shared_ptr<ExtraTypeInfo> ExtraTypeInfo::Copy() const {
-	return make_shared_ptr<ExtraTypeInfo>(*this);
-}
 
 static bool CompareModifiers(const vector<Value> &left, const vector<Value> &right) {
 	// Check if the common prefix of the properties is the same for both types
-	auto common_props = MinValue(left.size(), right.size());
+	const auto common_props = MinValue(left.size(), right.size());
 	for (idx_t i = 0; i < common_props; i++) {
 		if (left[i].type() != right[i].type()) {
 			return false;
@@ -34,13 +25,65 @@ static bool CompareModifiers(const vector<Value> &left, const vector<Value> &rig
 		if (left[i].IsNull() || right[i].IsNull()) {
 			continue;
 		}
+
 		if (left[i] != right[i]) {
 			return false;
 		}
 	}
 	return true;
 }
 
+bool ExtensionTypeInfo::Equals(optional_ptr<ExtensionTypeInfo> other_p) const {
+	if (other_p == nullptr) {
+		return false;
+	}
+	if (!CompareModifiers(modifiers, other_p->modifiers)) {
+		return false;
+	}
+
+	// Properties are optional, so only compare those present in both
+	for (auto &kv : properties) {
+		auto it = other_p->properties.find(kv.first);
+		if (it == other_p->properties.end()) {
+			continue;
+		}
+		if (kv.second != it->second) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+//===--------------------------------------------------------------------===//
+// Extra Type Info
+//===--------------------------------------------------------------------===//
+ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type) : type(type) {
+}
+ExtraTypeInfo::ExtraTypeInfo(ExtraTypeInfoType type, string alias) : type(type), alias(std::move(alias)) {
+}
+ExtraTypeInfo::~ExtraTypeInfo() {
+}
+
+ExtraTypeInfo::ExtraTypeInfo(const ExtraTypeInfo &other) : type(other.type), alias(other.alias) {
+	if (other.extension_info) {
+		extension_info = make_uniq<ExtensionTypeInfo>(*other.extension_info);
+	}
+}
+
+ExtraTypeInfo &ExtraTypeInfo::operator=(const ExtraTypeInfo &other) {
+	type = other.type;
+	alias = other.alias;
+	if (other.extension_info) {
+		extension_info = make_uniq<ExtensionTypeInfo>(*other.extension_info);
+	}
+	return *this;
+}
+
+shared_ptr<ExtraTypeInfo> ExtraTypeInfo::Copy() const {
+	return shared_ptr<ExtraTypeInfo>(new ExtraTypeInfo(*this));
+}
+
 bool ExtraTypeInfo::Equals(ExtraTypeInfo *other_p) const {
 	if (type == ExtraTypeInfoType::INVALID_TYPE_INFO || type == ExtraTypeInfoType::STRING_TYPE_INFO ||
 	    type == ExtraTypeInfoType::GENERIC_TYPE_INFO) {
@@ -54,7 +97,7 @@ bool ExtraTypeInfo::Equals(ExtraTypeInfo *other_p) const {
 		if (alias != other_p->alias) {
 			return false;
 		}
-		if (!CompareModifiers(modifiers, other_p->modifiers)) {
+		if (extension_info && !extension_info->Equals(other_p->extension_info.get())) {
 			return false;
 		}
 		return true;
@@ -68,7 +111,7 @@ bool ExtraTypeInfo::Equals(ExtraTypeInfo *other_p) const {
 	if (alias != other_p->alias) {
 		return false;
 	}
-	if (!CompareModifiers(modifiers, other_p->modifiers)) {
+	if (extension_info && !extension_info->Equals(other_p->extension_info.get())) {
 		return false;
 	}
 	return EqualsInternal(other_p);

src/duckdb/src/common/types.cpp

@@ -365,21 +365,29 @@ bool TypeIsInteger(PhysicalType type) {
 	       type == PhysicalType::UINT128;
 }
 
+static string TypeModifierListToString(const vector<Value> &mod_list) {
+	string result;
+	if (mod_list.empty()) {
+		return result;
+	}
+	result = "(";
+	for (idx_t i = 0; i < mod_list.size(); i++) {
+		result += mod_list[i].ToString();
+		if (i < mod_list.size() - 1) {
+			result += ", ";
+		}
+	}
+	result += ")";
+	return result;
+}
+
 string LogicalType::ToString() const {
 	if (id_ != LogicalTypeId::USER) {
 		auto alias = GetAlias();
 		if (!alias.empty()) {
-			auto mods_ptr = GetModifiers();
-			if (mods_ptr && !mods_ptr->empty()) {
-				auto &mods = *mods_ptr;
-				alias += "(";
-				for (idx_t i = 0; i < mods.size(); i++) {
-					alias += mods[i].ToString();
-					if (i < mods.size() - 1) {
-						alias += ", ";
-					}
-				}
-				alias += ")";
+			if (HasExtensionInfo()) {
+				auto &ext_info = *GetExtensionInfo();
+				alias += TypeModifierListToString(ext_info.modifiers);
 			}
 			return alias;
 		}
@@ -491,14 +499,7 @@ string LogicalType::ToString() const {
 		result += KeywordHelper::WriteOptionallyQuoted(type);
 
 		if (!mods.empty()) {
-			result += "(";
-			for (idx_t i = 0; i < mods.size(); i++) {
-				result += mods[i].ToString();
-				if (i < mods.size() - 1) {
-					result += ", ";
-				}
-			}
-			result += ")";
+			result += TypeModifierListToString(mods);
 		}
 
 		return result;
@@ -1360,51 +1361,32 @@ bool LogicalType::HasAlias() const {
 	return false;
 }
 
-void LogicalType::SetModifiers(vector<Value> modifiers) {
-	if (!type_info_ && !modifiers.empty()) {
-		type_info_ = make_shared_ptr<ExtraTypeInfo>(ExtraTypeInfoType::GENERIC_TYPE_INFO);
-	}
-	type_info_->modifiers = std::move(modifiers);
-}
-
-bool LogicalType::HasModifiers() const {
-	if (id() == LogicalTypeId::USER) {
-		return !UserType::GetTypeModifiers(*this).empty();
-	}
-	if (type_info_) {
-		return !type_info_->modifiers.empty();
+bool LogicalType::HasExtensionInfo() const {
+	if (type_info_ && type_info_->extension_info) {
+		return true;
 	}
 	return false;
 }
 
-vector<Value> LogicalType::GetModifiersCopy() const {
-	if (id() == LogicalTypeId::USER) {
-		return UserType::GetTypeModifiers(*this);
-	}
-	if (type_info_) {
-		return type_info_->modifiers;
+optional_ptr<const ExtensionTypeInfo> LogicalType::GetExtensionInfo() const {
+	if (type_info_ && type_info_->extension_info) {
+		return type_info_->extension_info.get();
 	}
-	return {};
+	return nullptr;
 }
 
-optional_ptr<vector<Value>> LogicalType::GetModifiers() {
-	if (id() == LogicalTypeId::USER) {
-		return UserType::GetTypeModifiers(*this);
-	}
-	if (type_info_) {
-		return type_info_->modifiers;
+optional_ptr<ExtensionTypeInfo> LogicalType::GetExtensionInfo() {
+	if (type_info_ && type_info_->extension_info) {
+		return type_info_->extension_info.get();
 	}
 	return nullptr;
 }
 
-optional_ptr<const vector<Value>> LogicalType::GetModifiers() const {
-	if (id() == LogicalTypeId::USER) {
-		return UserType::GetTypeModifiers(*this);
-	}
-	if (type_info_) {
-		return type_info_->modifiers;
+void LogicalType::SetExtensionInfo(unique_ptr<ExtensionTypeInfo> info) {
+	if (!type_info_) {
+		type_info_ = make_shared_ptr<ExtraTypeInfo>(ExtraTypeInfoType::GENERIC_TYPE_INFO);
 	}
-	return nullptr;
+	type_info_->extension_info = std::move(info);
 }
 
 //===--------------------------------------------------------------------===//

src/duckdb/src/execution/operator/persistent/physical_copy_database.cpp

@@ -42,6 +42,7 @@ SourceResultType PhysicalCopyDatabase::GetData(ExecutionContext &context, DataCh
 			catalog.CreateType(context.client, create_info->Cast<CreateTypeInfo>());
 			break;
 		case CatalogType::MACRO_ENTRY:
+		case CatalogType::TABLE_MACRO_ENTRY:
 			catalog.CreateFunction(context.client, create_info->Cast<CreateMacroInfo>());
 			break;
 		case CatalogType::TABLE_ENTRY: {

src/duckdb/src/execution/physical_plan/plan_get.cpp

@@ -160,8 +160,9 @@ unique_ptr<PhysicalOperator> PhysicalPlanGenerator::CreatePlan(LogicalGet &op) {
 		vector<unique_ptr<Expression>> expressions;
 		for (auto &column_id : column_ids) {
 			if (column_id.IsRowIdColumn()) {
-				types.emplace_back(LogicalType::ROW_TYPE);
-				expressions.push_back(make_uniq<BoundConstantExpression>(Value::BIGINT(0)));
+				types.emplace_back(op.GetRowIdType());
+				// Now how to make that a constant expression.
+				expressions.push_back(make_uniq<BoundConstantExpression>(Value(op.GetRowIdType())));
 			} else {
 				auto col_id = column_id.GetPrimaryIndex();
 				auto type = op.returned_types[col_id];

src/duckdb/src/function/table/arrow_conversion.cpp

@@ -1340,17 +1340,31 @@ static void ColumnArrowToDuckDBDictionary(Vector &vector, ArrowArray &array, Arr
 }
 
 void ArrowTableFunction::ArrowToDuckDB(ArrowScanLocalState &scan_state, const arrow_column_map_t &arrow_convert_data,
-                                       DataChunk &output, idx_t start, bool arrow_scan_is_projected) {
+                                       DataChunk &output, idx_t start, bool arrow_scan_is_projected,
+                                       idx_t rowid_column_index) {
 	for (idx_t idx = 0; idx < output.ColumnCount(); idx++) {
 		auto col_idx = scan_state.column_ids.empty() ? idx : scan_state.column_ids[idx];
 
 		// If projection was not pushed down into the arrow scanner, but projection pushdown is enabled on the
 		// table function, we need to use original column ids here.
 		auto arrow_array_idx = arrow_scan_is_projected ? idx : col_idx;
 
-		if (col_idx == COLUMN_IDENTIFIER_ROW_ID) {
-			// This column is skipped by the projection pushdown
-			continue;
+		if (rowid_column_index != COLUMN_IDENTIFIER_ROW_ID) {
+			if (col_idx == COLUMN_IDENTIFIER_ROW_ID) {
+				arrow_array_idx = rowid_column_index;
+			} else if (col_idx >= rowid_column_index) {
+				// Since the rowid column is skipped when the table is bound (its not a named column),
+				// we need to shift references forward in the Arrow array by one to match the alignment
+				// that DuckDB believes the Arrow array is in.
+				col_idx += 1;
+				arrow_array_idx += 1;
+			}
+		} else {
+			// If there isn't any defined row_id_index, and we're asked for it, skip the column.
+			// This is the incumbent behavior.
+			if (col_idx == COLUMN_IDENTIFIER_ROW_ID) {
+				continue;
+			}
 		}
 
 		auto &parent_array = scan_state.chunk->arrow_array;

src/duckdb/src/function/table/version/pragma_version.cpp

@@ -1,5 +1,5 @@
 #ifndef DUCKDB_PATCH_VERSION
-#define DUCKDB_PATCH_VERSION "4-dev3961"
+#define DUCKDB_PATCH_VERSION "4-dev3985"
 #endif
 #ifndef DUCKDB_MINOR_VERSION
 #define DUCKDB_MINOR_VERSION 1
@@ -8,10 +8,10 @@
 #define DUCKDB_MAJOR_VERSION 1
 #endif
 #ifndef DUCKDB_VERSION
-#define DUCKDB_VERSION "v1.1.4-dev3961"
+#define DUCKDB_VERSION "v1.1.4-dev3985"
 #endif
 #ifndef DUCKDB_SOURCE_ID
-#define DUCKDB_SOURCE_ID "4488c61ee7"
+#define DUCKDB_SOURCE_ID "adc6f607a7"
 #endif
 #include "duckdb/function/table/system_functions.hpp"
 #include "duckdb/main/database.hpp"

src/duckdb/src/include/duckdb/catalog/catalog_entry/table_catalog_entry.hpp

@@ -117,6 +117,11 @@ class TableCatalogEntry : public StandardEntry {
 	//! Returns true, if the table has a primary key, else false.
 	bool HasPrimaryKey() const;
 
+	//! Returns the rowid type of this table
+	virtual LogicalType GetRowIdType() const {
+		return LogicalType::ROW_TYPE;
+	}
+
 protected:
 	//! A list of columns that are part of this table
 	ColumnList columns;

src/duckdb/src/include/duckdb/common/extension_type_info.hpp

@@ -0,0 +1,19 @@
+#pragma once
+
+#include "duckdb/common/string.hpp"
+#include "duckdb/common/types/value.hpp"
+#include "duckdb/common/serializer/serializer.hpp"
+
+namespace duckdb {
+
+struct ExtensionTypeInfo {
+	vector<Value> modifiers;
+	unordered_map<string, Value> properties;
+
+public:
+	void Serialize(Serializer &serializer) const;
+	static unique_ptr<ExtensionTypeInfo> Deserialize(Deserializer &source);
+	bool Equals(optional_ptr<ExtensionTypeInfo> other_p) const;
+};
+
+} // namespace duckdb

src/duckdb/src/include/duckdb/common/extra_type_info.hpp

@@ -10,6 +10,7 @@
 
 #include "duckdb/common/common.hpp"
 #include "duckdb/common/types/vector.hpp"
+#include "duckdb/common/extension_type_info.hpp"
 
 namespace duckdb {
 
@@ -30,13 +31,18 @@ enum class ExtraTypeInfoType : uint8_t {
 };
 
 struct ExtraTypeInfo {
+	ExtraTypeInfoType type;
+	string alias;
+	unique_ptr<ExtensionTypeInfo> extension_info;
+
 	explicit ExtraTypeInfo(ExtraTypeInfoType type);
 	explicit ExtraTypeInfo(ExtraTypeInfoType type, string alias);
 	virtual ~ExtraTypeInfo();
 
-	ExtraTypeInfoType type;
-	string alias;
-	vector<Value> modifiers;
+protected:
+	// copy	constructor (protected)
+	ExtraTypeInfo(const ExtraTypeInfo &other);
+	ExtraTypeInfo &operator=(const ExtraTypeInfo &other);
 
 public:
 	bool Equals(ExtraTypeInfo *other_p) const;

src/duckdb/src/include/duckdb/common/types.hpp

@@ -234,6 +234,7 @@ enum class LogicalTypeId : uint8_t {
 };
 
 struct ExtraTypeInfo;
+struct ExtensionTypeInfo;
 
 struct aggregate_state_t; // NOLINT: mimic std casing
 
@@ -323,11 +324,11 @@ struct LogicalType {
 	DUCKDB_API void SetAlias(string alias);
 	DUCKDB_API bool HasAlias() const;
 	DUCKDB_API string GetAlias() const;
-	DUCKDB_API void SetModifiers(vector<Value> modifiers);
-	DUCKDB_API bool HasModifiers() const;
-	DUCKDB_API vector<Value> GetModifiersCopy() const;
-	DUCKDB_API optional_ptr<vector<Value>> GetModifiers();
-	DUCKDB_API optional_ptr<const vector<Value>> GetModifiers() const;
+
+	DUCKDB_API bool HasExtensionInfo() const;
+	DUCKDB_API optional_ptr<const ExtensionTypeInfo> GetExtensionInfo() const;
+	DUCKDB_API optional_ptr<ExtensionTypeInfo> GetExtensionInfo();
+	DUCKDB_API void SetExtensionInfo(unique_ptr<ExtensionTypeInfo> info);
 
 	//! Returns the maximum logical type when combining the two types - or throws an exception if combining is not possible
 	DUCKDB_API static LogicalType MaxLogicalType(ClientContext &context, const LogicalType &left, const LogicalType &right);