Skip to content

Commit

Permalink
[Refactor](function) make all Datetime arithmetic operation overflow …
Browse files Browse the repository at this point in the history
…lead to exception in BE (#45265)

### What problem does this PR solve?

Issue Number: close #xxx

Related PR: #xxx

Problem Summary:

1. totally refactored `FunctionDateOrDateTimeComputation`. removed some
unnecessary function and template. simplified some template
calculations.
2. All Datetime arithmetic operation overflow will lead to exception
now. before for nullable input it will get `NULL` result
see: 
```sql
mysql> select date_add('5000-10-10', interval 10000 year);
+------------------------------------------------+
| years_add(cast('5000-10-10' as DATEV2), 10000) |
+------------------------------------------------+
| NULL                                           |
+------------------------------------------------+
1 row in set (0.10 sec)
```
now:
```sql
ERROR 1105 (HY000): errCode = 2, detailMessage = (xxx)[E-218][E-218] Operation years_add of 5000-10-10, 10000 out of range
```

### Release note

All Datetime arithmetic operation overflow will lead to exception now.

### Check List (For Author)

- Test <!-- At least one of them must be included. -->
    - [ ] Regression test
    - [ ] Unit Test
    - [ ] Manual test (add detailed scripts or steps below)
    - [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
        - [x] Previous test can cover this change.
        - [ ] No code files have been changed.
        - [ ] Other reason <!-- Add your reason?  -->

- Behavior changed:
    - [ ] No.
    - [x] Yes. <!-- Explain the behavior change -->

- Does this need documentation?
    - [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
apache/doris-website#1214 -->

### Check List (For Reviewer who merge this PR)

- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
  • Loading branch information
zclllyybb authored Dec 23, 2024
1 parent 208fde0 commit 014f84a
Show file tree
Hide file tree
Showing 11 changed files with 471 additions and 575 deletions.
7 changes: 5 additions & 2 deletions be/src/util/datetype_cast.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@
/*
* We use these function family to clarify our types of datelike type. for example:
* DataTypeDate -------------------> ColumnDate -----------------------> Int64
* | TypeToColumn ValueTypeOfColumn
* | TypeToValueType
* | | TypeToColumn ValueTypeOfColumn |
* | ↘--------------------------------------------------------------↗
* | ::FieldType
* ↓ TypeToValueType
* VecDateTimeValue
*/
namespace doris::date_cast {
Expand Down Expand Up @@ -102,6 +104,7 @@ constexpr bool IsV1() {
std::is_same_v<Type, vectorized::Int64>);
}

// only for datelike types.
template <typename Type>
constexpr bool IsV2() {
return !IsV1<Type>();
Expand Down
3 changes: 0 additions & 3 deletions be/src/vec/common/typeid_cast.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,11 @@

#pragma once

#include <string>
#include <type_traits>
#include <typeindex>
#include <typeinfo>

#include "common/exception.h"
#include "common/status.h"
#include "vec/common/demangle.h"

/** Checks type by comparing typeid.
* The exact match of the type is checked. That is, cast to the ancestor will be unsuccessful.
Expand Down
11 changes: 5 additions & 6 deletions be/src/vec/functions/array/function_array_range.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
// under the License.

#include <glog/logging.h>
#include <stddef.h>

#include <algorithm>
#include <boost/iterator/iterator_facade.hpp>
#include <cstddef>
#include <memory>
#include <utility>

Expand All @@ -41,11 +41,11 @@
#include "vec/data_types/data_type_date_time.h"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_time_v2.h"
#include "vec/functions/function.h"
#include "vec/functions/function_date_or_datetime_computation.h"
#include "vec/functions/simple_function_factory.h"
#include "vec/runtime/vdatetime_value.h"
#include "vec/utils/util.hpp"

namespace doris {
class FunctionContext;
Expand Down Expand Up @@ -229,10 +229,9 @@ struct RangeImplUtil {
dest_nested_null_map.push_back(0);
offset++;
move++;
idx = doris::vectorized::date_time_add<
UNIT::value, DateV2Value<DateTimeV2ValueType>,
DateV2Value<DateTimeV2ValueType>, DateTimeV2>(idx, step_row,
is_null);
idx = doris::vectorized::date_time_add<UNIT::value, DataTypeDateTimeV2,
DataTypeDateTimeV2>(idx, step_row,
is_null);
}
dest_offsets.push_back(offset);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ using FunctionWeeksDiff =
using FunctionHoursDiff =
FunctionDateOrDateTimeComputation<HoursDiffImpl<DataTypeDateTime, DataTypeDateTime>>;
using FunctionMinutesDiff =
FunctionDateOrDateTimeComputation<MintueSDiffImpl<DataTypeDateTime, DataTypeDateTime>>;
FunctionDateOrDateTimeComputation<MintuesDiffImpl<DataTypeDateTime, DataTypeDateTime>>;
using FunctionSecondsDiff =
FunctionDateOrDateTimeComputation<SecondsDiffImpl<DataTypeDateTime, DataTypeDateTime>>;

Expand All @@ -68,6 +68,7 @@ struct NowFunctionName {
static constexpr auto name = "now";
};

//TODO: remove the inter-layer CurrentDateTimeImpl
using FunctionNow = FunctionCurrentDateOrDateTime<CurrentDateTimeImpl<NowFunctionName, false>>;

using FunctionNowWithPrecision =
Expand Down
832 changes: 319 additions & 513 deletions be/src/vec/functions/function_date_or_datetime_computation.h

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -95,14 +95,14 @@ using FunctionDatetimeV2SubYears =
FUNCTION_DATEV2_WITH_TWO_ARGS(NAME, IMPL, DataTypeDateTimeV2, DataTypeDateV2) \
FUNCTION_DATEV2_WITH_TWO_ARGS(NAME, IMPL, DataTypeDateV2, DataTypeDateTimeV2) \
FUNCTION_DATEV2_WITH_TWO_ARGS(NAME, IMPL, DataTypeDateV2, DataTypeDateV2)

// these diff functions accept all v2 types. but for v1 only datetime.
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2DateDiff, DateDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2TimeDiff, TimeDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2YearsDiff, YearsDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2MonthsDiff, MonthsDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2WeeksDiff, WeeksDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2HoursDiff, HoursDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2MinutesDiff, MintueSDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2MinutesDiff, MintuesDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2SecondsDiff, SecondsDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2DaysDiff, DaysDiffImpl)
ALL_FUNCTION_DATEV2_WITH_TWO_ARGS(FunctionDatetimeV2MilliSecondsDiff, MilliSecondsDiffImpl)
Expand Down
18 changes: 11 additions & 7 deletions be/src/vec/functions/function_helpers.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,8 @@

#pragma once

#include <stddef.h>

#include <cstddef>
#include <tuple>
#include <type_traits>

#include "vec/columns/column.h"
#include "vec/columns/column_const.h"
Expand Down Expand Up @@ -53,11 +51,15 @@ const Type* check_and_get_data_type(const IDataType* data_type) {

template <typename Type>
const ColumnConst* check_and_get_column_const(const IColumn* column) {
if (!column || !is_column_const(*column)) return {};
if (!column || !is_column_const(*column)) {
return nullptr;
}

const ColumnConst* res = assert_cast<const ColumnConst*, TypeCheckOnRelease::DISABLE>(column);
const auto* res = assert_cast<const ColumnConst*, TypeCheckOnRelease::DISABLE>(column);

if (!check_column<Type>(&res->get_data_column())) return {};
if (!check_column<Type>(&res->get_data_column())) {
return nullptr;
}

return res;
}
Expand All @@ -66,7 +68,9 @@ template <typename Type>
const Type* check_and_get_column_constData(const IColumn* column) {
const ColumnConst* res = check_and_get_column_const<Type>(column);

if (!res) return {};
if (!res) {
return nullptr;
}

return static_cast<const Type*>(&res->get_data_column());
}
Expand Down
2 changes: 1 addition & 1 deletion be/test/vec/function/function_test_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ using Row = std::pair<CellSet, Expect>;
using DataSet = std::vector<Row>;
using InputTypeSet = std::vector<AnyType>;

// FIXME: should use exception or expected to deal null value.w
// FIXME: should use exception or expected to deal null value.
int64_t str_to_date_time(std::string datetime_str, bool data_time = true);
uint32_t str_to_date_v2(std::string datetime_str, std::string datetime_format);
uint64_t str_to_datetime_v2(std::string datetime_str, std::string datetime_format);
Expand Down
82 changes: 64 additions & 18 deletions be/test/vec/function/function_time_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
// specific language governing permissions and limitations
// under the License.

#include <gtest/gtest.h>
#include <stdint.h>

#include <iomanip>
Expand Down Expand Up @@ -299,29 +300,45 @@ TEST(VTimestampFunctionsTest, years_add_test) {

InputTypeSet input_types = {TypeIndex::DateTime, TypeIndex::Int32};

DataSet data_set = {
{{std::string("2020-05-23 00:00:00"), 5}, str_to_date_time("2025-05-23 00:00:00")},
{{std::string("2020-05-23 00:00:00"), -5}, str_to_date_time("2015-05-23 00:00:00")},
{{std::string(""), 5}, Null()},
{{std::string("2020-05-23 00:00:00"), 8000}, Null()},
{{Null(), 5}, Null()}};
{
DataSet data_set = {
{{std::string("2020-05-23 00:00:00"), 5}, str_to_date_time("2025-05-23 00:00:00")},
{{std::string("2020-05-23 00:00:00"), -5}, str_to_date_time("2015-05-23 00:00:00")},
{{std::string(""), 5}, Null()},
{{Null(), 5}, Null()}};

static_cast<void>(check_function<DataTypeDateTime, true>(func_name, input_types, data_set));
static_cast<void>(check_function<DataTypeDateTime, true>(func_name, input_types, data_set));
}

{
DataSet data_set = {{{std::string("2020-05-23 00:00:00"), 8000}, Null()}};

EXPECT_ANY_THROW(static_cast<void>(
check_function<DataTypeDateTime, true>(func_name, input_types, data_set)));
}
}

TEST(VTimestampFunctionsTest, years_sub_test) {
std::string func_name = "years_sub";

InputTypeSet input_types = {TypeIndex::DateTime, TypeIndex::Int32};

DataSet data_set = {
{{std::string("2020-05-23 00:00:00"), 5}, str_to_date_time("2015-05-23 00:00:00")},
{{std::string("2020-05-23 00:00:00"), -5}, str_to_date_time("2025-05-23 00:00:00")},
{{std::string(""), 5}, Null()},
{{std::string("2020-05-23 00:00:00"), 3000}, Null()},
{{Null(), 5}, Null()}};
{
DataSet data_set = {
{{std::string("2020-05-23 00:00:00"), 5}, str_to_date_time("2015-05-23 00:00:00")},
{{std::string("2020-05-23 00:00:00"), -5}, str_to_date_time("2025-05-23 00:00:00")},
{{std::string(""), 5}, Null()},
{{Null(), 5}, Null()}};

static_cast<void>(check_function<DataTypeDateTime, true>(func_name, input_types, data_set));
static_cast<void>(check_function<DataTypeDateTime, true>(func_name, input_types, data_set));
}

{
DataSet data_set = {{{std::string("2020-05-23 00:00:00"), 3000}, Null()}};

EXPECT_ANY_THROW(static_cast<void>(
check_function<DataTypeDateTime, true>(func_name, input_types, data_set)));
}
}

TEST(VTimestampFunctionsTest, months_add_test) {
Expand Down Expand Up @@ -1043,11 +1060,18 @@ TEST(VTimestampFunctionsTest, years_add_v2_test) {
{{std::string("2020-05-23"), 5}, str_to_date_v2("2025-05-23", "%Y-%m-%d")},
{{std::string("2020-05-23"), -5}, str_to_date_v2("2015-05-23", "%Y-%m-%d")},
{{std::string(""), 5}, Null()},
{{std::string("2020-05-23"), 8000}, Null()},
{{Null(), 5}, Null()}};

static_cast<void>(check_function<DataTypeDateV2, true>(func_name, input_types, data_set));
}
{
InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};

DataSet data_set = {{{std::string("2020-05-23"), 8000}, Null()}};

EXPECT_ANY_THROW(static_cast<void>(
check_function<DataTypeDateV2, true>(func_name, input_types, data_set)));
}

{
InputTypeSet input_types = {TypeIndex::DateTimeV2, TypeIndex::Int32};
Expand All @@ -1057,12 +1081,19 @@ TEST(VTimestampFunctionsTest, years_add_v2_test) {
{{std::string("2020-05-23 00:00:11.123"), -5},
str_to_datetime_v2("2015-05-23 00:00:11.123", "%Y-%m-%d %H:%i:%s.%f")},
{{std::string(""), 5}, Null()},
{{std::string("2020-05-23 00:00:11.123"), 8000}, Null()},
{{Null(), 5}, Null()}};

static_cast<void>(
check_function<DataTypeDateTimeV2, true>(func_name, input_types, data_set));
}
{
InputTypeSet input_types = {TypeIndex::DateTimeV2, TypeIndex::Int32};

DataSet data_set = {{{std::string("2020-05-23 00:00:11.123"), 8000}, Null()}};

EXPECT_ANY_THROW(static_cast<void>(
check_function<DataTypeDateTimeV2, true>(func_name, input_types, data_set)));
}
}

TEST(VTimestampFunctionsTest, years_sub_v2_test) {
Expand All @@ -1075,11 +1106,19 @@ TEST(VTimestampFunctionsTest, years_sub_v2_test) {
{{std::string("2020-05-23"), 5}, str_to_date_v2("2015-05-23", "%Y-%m-%d")},
{{std::string("2020-05-23"), -5}, str_to_date_v2("2025-05-23", "%Y-%m-%d")},
{{std::string(""), 5}, Null()},
{{std::string("2020-05-23"), 3000}, Null()},
{{Null(), 5}, Null()}};

static_cast<void>(check_function<DataTypeDateV2, true>(func_name, input_types, data_set));
}
{
InputTypeSet input_types = {TypeIndex::DateV2, TypeIndex::Int32};

DataSet data_set = {{{std::string("2020-05-23"), 3000}, Null()}};

EXPECT_ANY_THROW(static_cast<void>(
check_function<DataTypeDateV2, true>(func_name, input_types, data_set)));
}

{
InputTypeSet input_types = {TypeIndex::DateTimeV2, TypeIndex::Int32};

Expand All @@ -1088,12 +1127,19 @@ TEST(VTimestampFunctionsTest, years_sub_v2_test) {
{{std::string("2020-05-23 00:00:11.123"), -5},
str_to_datetime_v2("2025-05-23 00:00:11.123", "%Y-%m-%d %H:%i:%s.%f")},
{{std::string(""), 5}, Null()},
{{std::string("2020-05-23 00:00:11.123"), 3000}, Null()},
{{Null(), 5}, Null()}};

static_cast<void>(
check_function<DataTypeDateTimeV2, true>(func_name, input_types, data_set));
}
{
InputTypeSet input_types = {TypeIndex::DateTimeV2, TypeIndex::Int32};

DataSet data_set = {{{std::string("2020-05-23 00:00:11.123"), 3000}, Null()}};

EXPECT_ANY_THROW(static_cast<void>(
check_function<DataTypeDateTimeV2, true>(func_name, input_types, data_set)));
}
}

TEST(VTimestampFunctionsTest, months_add_v2_test) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,6 @@ suite("test_date_function_const") {

test {
sql """select date_add("1900-01-01 12:00:00.123456", interval 10000000000 month);"""
exception "Operation months_add 133705200962757184 1410065408 out of range"
exception "Operation months_add of 1900-01-01 12:00:00.123456, 1410065408 out of range"
}
}
Loading

0 comments on commit 014f84a

Please sign in to comment.