Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Flatten #2102

Draft
wants to merge 3 commits into
base: master
Choose a base branch
from
Draft

Flatten #2102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d1cee66
Add `leaves_only` support to `to_arrow()`
texodus Mar 22, 2022
fa20270
WIP
texodus Apr 6, 2022
c77b664
WIP
texodus Apr 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cpp/perspective/src/cpp/context_grouped_pkey.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ t_ctx_grouped_pkey::sort_by(const std::vector<t_sortspec>& sortby) {
if (m_sortby.empty()) {
return;
}
m_traversal->sort_by(m_config, sortby, *this);
m_traversal->sort_by(sortby, *this);
}

void
Expand Down Expand Up @@ -642,7 +642,7 @@ t_ctx_grouped_pkey::rebuild() {
set_expansion_state(expansion_state);

if (!m_sortby.empty()) {
m_traversal->sort_by(m_config, m_sortby, *this);
m_traversal->sort_by(m_sortby, *this);
}
}

Expand Down
20 changes: 19 additions & 1 deletion cpp/perspective/src/cpp/context_one.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ t_ctx1::sort_by(const std::vector<t_sortspec>& sortby) {
if (m_sortby.empty()) {
return;
}
m_traversal->sort_by(m_config, sortby, *(m_tree.get()));
m_traversal->sort_by(sortby, *(m_tree.get()));
}

void
Expand Down Expand Up @@ -519,6 +519,24 @@ t_ctx1::reset_step_state() {
}
}

std::shared_ptr<t_traversal>
t_ctx1::set_is_leaves_only() {
auto trav = m_traversal;
m_traversal = std::make_shared<t_traversal>(m_tree);
m_traversal->set_is_leaves_only(true);
if (!m_sortby.empty()) {
m_traversal->sort_by(m_sortby, *(m_tree.get()));
}

m_traversal->set_depth(m_sortby, m_config.get_num_rpivots() - 1);
return trav;
}

void
t_ctx1::clear_is_leaves_only(std::shared_ptr<t_traversal> trav) {
m_traversal = trav;
}

t_index
t_ctx1::sidedness() const {
return 1;
Expand Down
22 changes: 20 additions & 2 deletions cpp/perspective/src/cpp/context_two.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,7 @@ void
t_ctx2::column_sort_by(const std::vector<t_sortspec>& sortby) {
PSP_TRACE_SENTINEL();
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
m_ctraversal->sort_by(m_config, sortby, *(ctree().get()));
m_ctraversal->sort_by(sortby, *(ctree().get()));
}

void
Expand All @@ -493,7 +493,7 @@ t_ctx2::sort_by(const std::vector<t_sortspec>& sortby) {
if (m_sortby.empty()) {
return;
}
m_rtraversal->sort_by(m_config, sortby, *(rtree().get()), this);
m_rtraversal->sort_by(sortby, *(rtree().get()), this);
}

void
Expand All @@ -503,6 +503,24 @@ t_ctx2::reset_sortby() {
m_sortby = std::vector<t_sortspec>();
}

std::shared_ptr<t_traversal>
t_ctx2::set_is_leaves_only() {
auto trav = m_rtraversal;
m_rtraversal = std::make_shared<t_traversal>(rtree());
m_rtraversal->set_is_leaves_only(true);
if (!m_sortby.empty()) {
m_rtraversal->sort_by(m_sortby, *(rtree().get()), this);
}

m_rtraversal->set_depth(m_sortby, m_config.get_num_rpivots() - 1);
return trav;
}

void
t_ctx2::clear_is_leaves_only(std::shared_ptr<t_traversal> trav) {
m_rtraversal = trav;
}

void
t_ctx2::notify(const t_data_table& flattened) {
for (t_uindex tree_idx = 0, loop_end = m_trees.size(); tree_idx < loop_end;
Expand Down
12 changes: 12 additions & 0 deletions cpp/perspective/src/cpp/data_slice.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ t_data_slice<CTX_T>::t_data_slice(std::shared_ptr<CTX_T> ctx,
, m_slice(slice)
, m_column_names(column_names) {
m_stride = m_end_col - m_start_col;
t_uindex ctx_nrows = m_ctx->get_row_count();
t_uindex ncols = m_ctx->get_column_count();
auto ext = sanitize_get_data_extents(
ctx_nrows, ncols, start_row, end_row, start_col, end_col);
m_start_row = ext.m_srow;
m_end_row = ext.m_erow;
}

template <typename CTX_T>
Expand All @@ -48,6 +54,12 @@ t_data_slice<CTX_T>::t_data_slice(std::shared_ptr<CTX_T> ctx,
, m_column_names(column_names)
, m_column_indices(column_indices) {
m_stride = m_end_col - m_start_col;
t_uindex ctx_nrows = m_ctx->get_row_count();
t_uindex ncols = m_ctx->get_column_count();
auto ext = sanitize_get_data_extents(
ctx_nrows, ncols, start_row, end_row, start_col, end_col);
m_start_row = ext.m_srow;
m_end_row = ext.m_erow;
}

template <typename CTX_T>
Expand Down
19 changes: 12 additions & 7 deletions cpp/perspective/src/cpp/emscripten.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,9 +203,10 @@ namespace binding {
template <typename CTX_T>
t_val
to_arrow(std::shared_ptr<View<CTX_T>> view, std::int32_t start_row,
std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) {
std::shared_ptr<std::string> s
= view->to_arrow(start_row, end_row, start_col, end_col, true);
std::int32_t end_row, std::int32_t start_col, std::int32_t end_col,
bool is_leaves_only) {
std::shared_ptr<std::string> s = view->to_arrow(
start_row, end_row, start_col, end_col, true, is_leaves_only);
return str_to_arraybuffer(s)["buffer"];
}

Expand Down Expand Up @@ -1922,7 +1923,8 @@ EMSCRIPTEN_BINDINGS(perspective) {
.function("get_sort", &View<t_ctxunit>::get_sort)
.function("get_step_delta", &View<t_ctxunit>::get_step_delta)
.function("get_column_dtype", &View<t_ctxunit>::get_column_dtype)
.function("is_column_only", &View<t_ctxunit>::is_column_only);
.function("is_column_only", &View<t_ctxunit>::is_column_only)
.function("set_separator", &View<t_ctxunit>::set_separator);

class_<View<t_ctx0>>("View_ctx0")
.constructor<std::shared_ptr<Table>, std::shared_ptr<t_ctx0>,
Expand All @@ -1948,7 +1950,8 @@ EMSCRIPTEN_BINDINGS(perspective) {
.function("get_sort", &View<t_ctx0>::get_sort)
.function("get_step_delta", &View<t_ctx0>::get_step_delta)
.function("get_column_dtype", &View<t_ctx0>::get_column_dtype)
.function("is_column_only", &View<t_ctx0>::is_column_only);
.function("is_column_only", &View<t_ctx0>::is_column_only)
.function("set_separator", &View<t_ctx0>::set_separator);

class_<View<t_ctx1>>("View_ctx1")
.constructor<std::shared_ptr<Table>, std::shared_ptr<t_ctx1>,
Expand Down Expand Up @@ -1977,7 +1980,8 @@ EMSCRIPTEN_BINDINGS(perspective) {
.function("get_sort", &View<t_ctx1>::get_sort)
.function("get_step_delta", &View<t_ctx1>::get_step_delta)
.function("get_column_dtype", &View<t_ctx1>::get_column_dtype)
.function("is_column_only", &View<t_ctx1>::is_column_only);
.function("is_column_only", &View<t_ctx1>::is_column_only)
.function("set_separator", &View<t_ctx1>::set_separator);

class_<View<t_ctx2>>("View_ctx2")
.constructor<std::shared_ptr<Table>, std::shared_ptr<t_ctx2>,
Expand Down Expand Up @@ -2007,7 +2011,8 @@ EMSCRIPTEN_BINDINGS(perspective) {
.function("get_row_path", &View<t_ctx2>::get_row_path)
.function("get_step_delta", &View<t_ctx2>::get_step_delta)
.function("get_column_dtype", &View<t_ctx2>::get_column_dtype)
.function("is_column_only", &View<t_ctx2>::is_column_only);
.function("is_column_only", &View<t_ctx2>::is_column_only)
.function("set_separator", &View<t_ctx2>::set_separator);

/******************************************************************************
*
Expand Down
63 changes: 45 additions & 18 deletions cpp/perspective/src/cpp/traversal.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ t_vdnode::t_vdnode(bool expanded, bool has_children)
, m_has_children(has_children) {}

t_traversal::t_traversal(std::shared_ptr<const t_stree> tree)
: m_tree(tree) {
: m_tree(tree)
, m_is_leaves_only(false) {
t_stnode_vec rchildren;
tree->get_child_nodes(0, rchildren);
populate_root_children(rchildren);
Expand Down Expand Up @@ -174,14 +175,22 @@ t_traversal::expand_node(
count += 1;
}

// Update node being expanded
exp_tvnode.m_expanded = !sorted_idx.empty();
exp_tvnode.m_ndesc += n_changed;
exp_tvnode.m_nchild = n_changed;

// insert children of node into the traversal
m_nodes->insert(
m_nodes->begin() + exp_idx + 1, children.begin(), children.end());
if (m_is_leaves_only) {
if (children.size() > 0) {
m_nodes->erase(m_nodes->begin() + exp_idx);
m_nodes->insert(
m_nodes->begin() + exp_idx, children.begin(), children.end());
}
} else {
// Update node being expanded
exp_tvnode.m_expanded = !sorted_idx.empty();
exp_tvnode.m_ndesc += n_changed;
exp_tvnode.m_nchild = n_changed;

m_nodes->insert(
m_nodes->begin() + exp_idx + 1, children.begin(), children.end());
}

// update ancestors about their new descendents
update_ancestors(exp_idx, n_changed);
Expand Down Expand Up @@ -280,7 +289,6 @@ t_traversal::update_sucessors(t_index nidx, t_index n_changed) {

while (c_node->m_depth > 0) {
t_index pidx = nidx - c_node->m_rel_pidx;

const t_tvnode& p_node = (*m_nodes)[pidx];

t_index p_nchild = p_node.m_nchild;
Expand All @@ -289,7 +297,7 @@ t_traversal::update_sucessors(t_index nidx, t_index n_changed) {
for (int i = 0; i < p_nchild; i++) {
t_index curr_cidx = pidx + coffset;
t_tvnode& child_node = (*m_nodes)[curr_cidx];
if (curr_cidx > nidx) {
if (!m_is_leaves_only && curr_cidx > nidx) {
child_node.m_rel_pidx += n_changed;
}
if (child_node.m_expanded) {
Expand All @@ -312,7 +320,11 @@ t_traversal::get_tree_index(t_index idx) const {

t_uindex
t_traversal::size() const {
return m_nodes->size();
if (m_is_leaves_only) {
return m_nodes->size() - 1;
} else {
return m_nodes->size();
}
}

t_depth
Expand Down Expand Up @@ -467,12 +479,17 @@ t_traversal::get_leaves(std::vector<t_index>& out_data) const {
}

void
t_traversal::get_child_indices(
t_index nidx, std::vector<std::pair<t_index, t_index>>& out_data) const {
t_traversal::get_child_indices(t_index nidx,
std::vector<std::pair<t_index, t_index>>& out_data, t_index count) const {
const t_tvnode& tvnode = (*m_nodes)[nidx];
t_index nchild = tvnode.m_nchild;
t_index coffset = 1;
t_index nchild;
if (count >= 0) {
nchild = count;
} else {
nchild = tvnode.m_nchild;
}

t_index coffset = m_is_leaves_only && nidx > 0 ? 0 : 1;
for (int i = 0; i < nchild; i++) {
t_index curr_cidx = nidx + coffset;
const t_tvnode& child_node = (*m_nodes)[curr_cidx];
Expand Down Expand Up @@ -506,7 +523,7 @@ t_traversal::get_num_tree_leaves(t_index idx) const {
void
t_traversal::post_order(t_index nidx, std::vector<t_index>& out_vec) {
std::vector<std::pair<t_index, t_index>> children;
get_child_indices(nidx, children);
get_child_indices(nidx, children, -1);

for (t_index idx = 0, loop_end = children.size(); idx < loop_end; ++idx) {
post_order(children[idx].first, out_vec);
Expand All @@ -526,9 +543,14 @@ t_traversal::set_depth(
while (pending.size() > 0) {
t_index curidx = pending.back();
pending.pop_back();
n_changed += expand_node(sortby, curidx, ctx2);
t_index _n_changed = expand_node(sortby, curidx, ctx2);
n_changed += _n_changed;
std::vector<std::pair<t_index, t_index>> children;
get_child_indices(curidx, children);
if (m_is_leaves_only && curidx != 0) {
get_child_indices(curidx, children, _n_changed);
} else {
get_child_indices(curidx, children, -1);
}
std::vector<t_index> collapse;
for (t_index idx = 0, loop_end = children.size(); idx < loop_end;
++idx) {
Expand Down Expand Up @@ -668,6 +690,11 @@ t_traversal::drop_tree_indices(const std::vector<t_uindex>& indices) {
}
}

void
t_traversal::set_is_leaves_only(bool is_leaves_only) {
m_is_leaves_only = is_leaves_only;
}

bool
t_traversal::is_valid_idx(t_index idx) const {
return idx > 0 && idx < t_index(size());
Expand Down
2 changes: 1 addition & 1 deletion cpp/perspective/src/cpp/tree_context_common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ ctx_get_flattened_tree(t_index idx, t_depth stop_depth, t_traversal& trav,
t_index ptidx = trav.get_tree_index(idx);
trav.set_depth(sortby, stop_depth);
if (!sortby.empty()) {
trav.sort_by(config, sortby, *(trav.get_tree()));
trav.sort_by(sortby, *(trav.get_tree()));
}
t_index new_tvidx = trav.tree_index_lookup(ptidx, idx);
return trav.get_flattened_tree(new_tvidx, stop_depth);
Expand Down
55 changes: 50 additions & 5 deletions cpp/perspective/src/cpp/view.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,10 +453,8 @@ View<t_ctx2>::get_data(t_uindex start_row, t_uindex end_row, t_uindex start_col,
std::vector<std::vector<t_tscalar>> cols;
bool is_sorted = m_sort.size() > 0;

if (is_column_only()) {
start_row += m_row_offset;
end_row += m_row_offset;
}
start_row += m_row_offset;
end_row += m_row_offset;

if (is_sorted) {
/**
Expand Down Expand Up @@ -531,12 +529,53 @@ View<t_ctx2>::get_data(t_uindex start_row, t_uindex end_row, t_uindex start_col,
template <typename CTX_T>
std::shared_ptr<std::string>
View<CTX_T>::to_arrow(std::int32_t start_row, std::int32_t end_row,
std::int32_t start_col, std::int32_t end_col, bool emit_group_by) const {
std::int32_t start_col, std::int32_t end_col, bool emit_group_by,
bool is_leaves_only) const {
std::shared_ptr<t_data_slice<CTX_T>> data_slice
= get_data(start_row, end_row, start_col, end_col);
return data_slice_to_arrow(data_slice, emit_group_by);
};

template <>
std::shared_ptr<std::string>
View<t_ctx1>::to_arrow(std::int32_t start_row, std::int32_t end_row,
std::int32_t start_col, std::int32_t end_col, bool emit_group_by,
bool is_leaves_only) const {
std::shared_ptr<t_traversal> guard;
if (is_leaves_only) {
guard = m_ctx->set_is_leaves_only();
}

std::shared_ptr<t_data_slice<t_ctx1>> data_slice
= get_data(start_row, end_row, start_col, end_col);
auto arr = data_slice_to_arrow(data_slice, emit_group_by);
if (is_leaves_only) {
m_ctx->clear_is_leaves_only(guard);
}

return arr;
};

template <>
std::shared_ptr<std::string>
View<t_ctx2>::to_arrow(std::int32_t start_row, std::int32_t end_row,
std::int32_t start_col, std::int32_t end_col, bool emit_group_by,
bool is_leaves_only) const {
std::shared_ptr<t_traversal> guard;
if (is_leaves_only) {
guard = m_ctx->set_is_leaves_only();
}

std::shared_ptr<t_data_slice<t_ctx2>> data_slice
= get_data(start_row, end_row, start_col, end_col);
auto arr = data_slice_to_arrow(data_slice, emit_group_by);
if (is_leaves_only) {
m_ctx->clear_is_leaves_only(guard);
}

return arr;
};

template <typename CTX_T>
std::shared_ptr<std::string>
View<CTX_T>::to_csv(std::int32_t start_row, std::int32_t end_row,
Expand Down Expand Up @@ -1312,6 +1351,12 @@ View<CTX_T>::get_event_loop_thread_id() const {
};
#endif

template <typename CTX_T>
void
View<CTX_T>::set_separator(std::string sep) {
m_separator = sep;
}

/******************************************************************************
*
* Private
Expand Down
Loading