Skip to content

Commit 0bcebe8

Browse files
Exp/compress candidates (ton-blockchain#942)
* Compress block candidates in validator-session * Compress blocks in full-node (disabled for now) --------- Co-authored-by: SpyCheese <[email protected]>
1 parent 9452c36 commit 0bcebe8

21 files changed

+547
-111
lines changed

CMake/FindLZ4.cmake

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
###############################################################################
2+
# Find LZ4
3+
#
4+
# This sets the following variables:
5+
# LZ4_FOUND - True if LZ4 was found.
6+
# LZ4_INCLUDE_DIRS - Directories containing the LZ4 include files.
7+
# LZ4_LIBRARIES - Libraries needed to use LZ4.
8+
# LZ4_LIBRARY - Library needed to use LZ4.
9+
# LZ4_LIBRARY_DIRS - Library needed to use LZ4.
10+
11+
find_package(PkgConfig REQUIRED)
12+
13+
# If found, LZ$_* variables will be defined
14+
pkg_check_modules(LZ4 REQUIRED liblz4)
15+
16+
if(NOT LZ4_FOUND)
17+
find_path(LZ4_INCLUDE_DIR lz4.h
18+
HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
19+
PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
20+
PATH_SUFFIXES include)
21+
22+
find_library(LZ4_LIBRARY
23+
NAMES lz4 lz4_static
24+
HINTS "${LZ4_ROOT}" "$ENV{LZ4_ROOT}"
25+
PATHS "$ENV{PROGRAMFILES}/lz4" "$ENV{PROGRAMW6432}/lz4"
26+
PATH_SUFFIXES lib)
27+
28+
if(LZ4_LIBRARY)
29+
set(LZ4_LIBRARIES ${LZ4_LIBRARY})
30+
get_filename_component(LZ4_LIBRARY_DIRS ${LZ4_LIBRARY} DIRECTORY)
31+
endif()
32+
else()
33+
find_library(LZ4_LIBRARY
34+
NAMES lz4 lz4_static
35+
PATHS ${LZ4_LIBRARY_DIRS}
36+
NO_DEFAULT_PATH)
37+
endif()
38+
39+
mark_as_advanced(LZ4_LIBRARY LZ4_INCLUDE_DIRS LZ4_LIBRARY_DIRS LZ4_LIBRARIES)

tdutils/CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ if (NOT DEFINED CMAKE_INSTALL_LIBDIR)
1515
endif()
1616

1717
find_package(PkgConfig REQUIRED)
18+
find_package(LZ4)
1819
if (NOT ZLIB_FOUND)
1920
pkg_check_modules(ZLIB zlib)
2021
endif()
@@ -280,6 +281,15 @@ if (TDUTILS_MIME_TYPE)
280281
)
281282
endif()
282283

284+
if (LZ4_FOUND)
285+
set(TD_HAVE_LZ4 1)
286+
set(TDUTILS_SOURCE
287+
${TDUTILS_SOURCE}
288+
td/utils/lz4.cpp
289+
td/utils/lz4.h
290+
)
291+
endif()
292+
283293
set(TDUTILS_TEST_SOURCE
284294
${CMAKE_CURRENT_SOURCE_DIR}/test/buffer.cpp
285295
${CMAKE_CURRENT_SOURCE_DIR}/test/ConcurrentHashMap.cpp
@@ -338,6 +348,11 @@ endif()
338348
if (CRC32C_FOUND)
339349
target_link_libraries(tdutils PRIVATE crc32c)
340350
endif()
351+
352+
if (LZ4_FOUND)
353+
target_link_libraries(tdutils PRIVATE ${LZ4_LIBRARIES})
354+
endif()
355+
341356
if (ABSL_FOUND)
342357
target_link_libraries_system(tdutils absl::flat_hash_map absl::flat_hash_set absl::hash)
343358
endif()

tdutils/td/utils/config.h.in

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#cmakedefine01 TD_HAVE_OPENSSL
44
#cmakedefine01 TD_HAVE_ZLIB
55
#cmakedefine01 TD_HAVE_CRC32C
6+
#cmakedefine01 TD_HAVE_LZ4
67
#cmakedefine01 TD_HAVE_COROUTINES
78
#cmakedefine01 TD_HAVE_ABSL
89
#cmakedefine01 TD_FD_DEBUG

tdutils/td/utils/lz4.cpp

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
This file is part of TON Blockchain Library.
3+
4+
TON Blockchain Library is free software: you can redistribute it and/or modify
5+
it under the terms of the GNU Lesser General Public License as published by
6+
the Free Software Foundation, either version 2 of the License, or
7+
(at your option) any later version.
8+
9+
TON Blockchain Library is distributed in the hope that it will be useful,
10+
but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
GNU Lesser General Public License for more details.
13+
14+
You should have received a copy of the GNU Lesser General Public License
15+
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
#include "td/utils/buffer.h"
18+
#include "td/utils/misc.h"
19+
#include <lz4.h>
20+
21+
namespace td {
22+
23+
td::BufferSlice lz4_compress(td::Slice data) {
24+
int size = narrow_cast<int>(data.size());
25+
int buf_size = LZ4_compressBound(size);
26+
td::BufferSlice compressed(buf_size);
27+
int compressed_size = LZ4_compress_default(data.data(), compressed.data(), size, buf_size);
28+
CHECK(compressed_size > 0);
29+
return td::BufferSlice{compressed.as_slice().substr(0, compressed_size)};
30+
}
31+
32+
td::Result<td::BufferSlice> lz4_decompress(td::Slice data, int max_decompressed_size) {
33+
TRY_RESULT(size, narrow_cast_safe<int>(data.size()));
34+
if (max_decompressed_size < 0) {
35+
return td::Status::Error("invalid max_decompressed_size");
36+
}
37+
td::BufferSlice decompressed(max_decompressed_size);
38+
int result = LZ4_decompress_safe(data.data(), decompressed.data(), size, max_decompressed_size);
39+
if (result < 0) {
40+
return td::Status::Error(PSTRING() << "lz4 decompression failed, error code: " << result);
41+
}
42+
if (result == max_decompressed_size) {
43+
return decompressed;
44+
}
45+
return td::BufferSlice{decompressed.as_slice().substr(0, result)};
46+
}
47+
48+
} // namespace td

tdutils/td/utils/lz4.h

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
This file is part of TON Blockchain Library.
3+
4+
TON Blockchain Library is free software: you can redistribute it and/or modify
5+
it under the terms of the GNU Lesser General Public License as published by
6+
the Free Software Foundation, either version 2 of the License, or
7+
(at your option) any later version.
8+
9+
TON Blockchain Library is distributed in the hope that it will be useful,
10+
but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
GNU Lesser General Public License for more details.
13+
14+
You should have received a copy of the GNU Lesser General Public License
15+
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
#pragma once
18+
19+
#include "td/utils/buffer.h"
20+
#include "td/utils/Status.h"
21+
22+
namespace td {
23+
24+
td::BufferSlice lz4_compress(td::Slice data);
25+
td::Result<td::BufferSlice> lz4_decompress(td::Slice data, int max_decompressed_size);
26+
27+
} // namespace td

tl/generate/scheme/ton_api.tl

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,7 @@ validatorSession.candidateId src:int256 root_hash:int256 file_hash:int256 collat
309309

310310
validatorSession.blockUpdate ts:long actions:(vector validatorSession.round.Message) state:int = validatorSession.BlockUpdate;
311311
validatorSession.candidate src:int256 round:int root_hash:int256 data:bytes collated_data:bytes = validatorSession.Candidate;
312+
validatorSession.compressedCandidate flags:# src:int256 round:int root_hash:int256 decompressed_size:int data:bytes = validatorSession.Candidate;
312313

313314
validatorSession.config catchain_idle_timeout:double catchain_max_deps:int round_candidates:int next_candidate_delay:double round_attempt_duration:int
314315
max_round_attempts:int max_block_size:int max_collated_data_size:int = validatorSession.Config;
@@ -385,9 +386,13 @@ tonNode.externalMessage data:bytes = tonNode.ExternalMessage;
385386

386387
tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode.NewShardBlock;
387388

389+
tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data;
390+
388391
tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
389392
signatures:(vector tonNode.blockSignature)
390393
proof:bytes data:bytes = tonNode.Broadcast;
394+
tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
395+
flags:# compressed:bytes = tonNode.Broadcast;
391396
tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast;
392397
tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast;
393398
tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast;
@@ -401,9 +406,8 @@ tonNode.keyBlocks blocks:(vector tonNode.blockIdExt) incomplete:Bool error:Bool
401406
ton.blockId root_cell_hash:int256 file_hash:int256 = ton.BlockId;
402407
ton.blockIdApprove root_cell_hash:int256 file_hash:int256 = ton.BlockId;
403408

404-
tonNode.dataList data:(vector bytes) = tonNode.DataList;
405-
406409
tonNode.dataFull id:tonNode.blockIdExt proof:bytes block:bytes is_link:Bool = tonNode.DataFull;
410+
tonNode.dataFullCompressed id:tonNode.blockIdExt flags:# compressed:bytes is_link:Bool = tonNode.DataFull;
407411
tonNode.dataFullEmpty = tonNode.DataFull;
408412

409413
tonNode.capabilities version:int capabilities:long = tonNode.Capabilities;
@@ -430,18 +434,13 @@ tonNode.getNextKeyBlockIds block:tonNode.blockIdExt max_size:int = tonNode.KeyBl
430434
tonNode.downloadNextBlockFull prev_block:tonNode.blockIdExt = tonNode.DataFull;
431435
tonNode.downloadBlockFull block:tonNode.blockIdExt = tonNode.DataFull;
432436
tonNode.downloadBlock block:tonNode.blockIdExt = tonNode.Data;
433-
tonNode.downloadBlocks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
434437
tonNode.downloadPersistentState block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt = tonNode.Data;
435438
tonNode.downloadPersistentStateSlice block:tonNode.blockIdExt masterchain_block:tonNode.blockIdExt offset:long max_size:long = tonNode.Data;
436439
tonNode.downloadZeroState block:tonNode.blockIdExt = tonNode.Data;
437440
tonNode.downloadBlockProof block:tonNode.blockIdExt = tonNode.Data;
438441
tonNode.downloadKeyBlockProof block:tonNode.blockIdExt = tonNode.Data;
439-
tonNode.downloadBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
440-
tonNode.downloadKeyBlockProofs blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
441442
tonNode.downloadBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
442443
tonNode.downloadKeyBlockProofLink block:tonNode.blockIdExt = tonNode.Data;
443-
tonNode.downloadBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
444-
tonNode.downloadKeyBlockProofLinks blocks:(vector tonNode.blockIdExt) = tonNode.DataList;
445444
tonNode.getArchiveInfo masterchain_seqno:int = tonNode.ArchiveInfo;
446445
tonNode.getArchiveSlice archive_id:long offset:long max_size:int = tonNode.Data;
447446

tl/generate/scheme/ton_api.tlo

216 Bytes
Binary file not shown.

validator-session/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@ if (NOT OPENSSL_FOUND)
55
endif()
66

77
set(VALIDATOR_SESSION_SOURCE
8+
candidate-serializer.cpp
89
persistent-vector.cpp
910
validator-session-description.cpp
1011
validator-session-state.cpp
1112
validator-session.cpp
1213
validator-session-round-attempt-state.cpp
1314

15+
candidate-serializer.h
1416
persistent-vector.h
1517
validator-session-description.h
1618
validator-session-description.hpp
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
This file is part of TON Blockchain Library.
3+
4+
TON Blockchain Library is free software: you can redistribute it and/or modify
5+
it under the terms of the GNU Lesser General Public License as published by
6+
the Free Software Foundation, either version 2 of the License, or
7+
(at your option) any later version.
8+
9+
TON Blockchain Library is distributed in the hope that it will be useful,
10+
but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
GNU Lesser General Public License for more details.
13+
14+
You should have received a copy of the GNU Lesser General Public License
15+
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
#pragma once
18+
#include "candidate-serializer.h"
19+
#include "tl-utils/tl-utils.hpp"
20+
#include "vm/boc.h"
21+
#include "td/utils/lz4.h"
22+
#include "validator-session-types.h"
23+
24+
namespace ton::validatorsession {
25+
26+
td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
27+
bool compression_enabled) {
28+
if (!compression_enabled) {
29+
return serialize_tl_object(block, true);
30+
}
31+
vm::BagOfCells boc1, boc2;
32+
TRY_STATUS(boc1.deserialize(block->data_));
33+
if (boc1.get_root_count() != 1) {
34+
return td::Status::Error("block candidate should have exactly one root");
35+
}
36+
std::vector<td::Ref<vm::Cell>> roots = {boc1.get_root_cell()};
37+
TRY_STATUS(boc2.deserialize(block->collated_data_));
38+
for (int i = 0; i < boc2.get_root_count(); ++i) {
39+
roots.push_back(boc2.get_root_cell(i));
40+
}
41+
TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2));
42+
td::BufferSlice compressed = td::lz4_compress(data);
43+
LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size()
44+
<< " -> " << compressed.size();
45+
return create_serialize_tl_object<ton_api::validatorSession_compressedCandidate>(
46+
0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed));
47+
}
48+
49+
td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
50+
bool compression_enabled,
51+
int max_decompressed_data_size) {
52+
if (!compression_enabled) {
53+
return fetch_tl_object<ton_api::validatorSession_candidate>(data, true);
54+
}
55+
TRY_RESULT(f, fetch_tl_object<ton_api::validatorSession_compressedCandidate>(data, true));
56+
if (f->decompressed_size_ > max_decompressed_data_size) {
57+
return td::Status::Error("decompressed size is too big");
58+
}
59+
TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_));
60+
if (decompressed.size() != (size_t)f->decompressed_size_) {
61+
return td::Status::Error("decompressed size mismatch");
62+
}
63+
TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed));
64+
if (roots.empty()) {
65+
return td::Status::Error("boc is empty");
66+
}
67+
TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31));
68+
roots.erase(roots.begin());
69+
TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31));
70+
LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> "
71+
<< block_data.size() + collated_data.size();
72+
return create_tl_object<ton_api::validatorSession_candidate>(f->src_, f->round_, f->root_hash_, std::move(block_data),
73+
std::move(collated_data));
74+
}
75+
76+
} // namespace ton::validatorsession
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
This file is part of TON Blockchain Library.
3+
4+
TON Blockchain Library is free software: you can redistribute it and/or modify
5+
it under the terms of the GNU Lesser General Public License as published by
6+
the Free Software Foundation, either version 2 of the License, or
7+
(at your option) any later version.
8+
9+
TON Blockchain Library is distributed in the hope that it will be useful,
10+
but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
GNU Lesser General Public License for more details.
13+
14+
You should have received a copy of the GNU Lesser General Public License
15+
along with TON Blockchain Library. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
#pragma once
18+
#include "ton/ton-types.h"
19+
#include "auto/tl/ton_api.h"
20+
21+
namespace ton::validatorsession {
22+
23+
td::Result<td::BufferSlice> serialize_candidate(const tl_object_ptr<ton_api::validatorSession_candidate> &block,
24+
bool compression_enabled);
25+
td::Result<tl_object_ptr<ton_api::validatorSession_candidate>> deserialize_candidate(td::Slice data,
26+
bool compression_enabled,
27+
int max_decompressed_data_size);
28+
29+
} // namespace ton::validatorsession

0 commit comments

Comments
 (0)