Skip to content

Commit 25c2dbb

Browse files
authored
[opt](rowset) Remote fetch rowsets to avoid -230 error when capturing rowsets (apache#3967)
1 parent 8fef4bd commit 25c2dbb

53 files changed

Lines changed: 1012 additions & 322 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎be/src/cloud/cloud_base_compaction.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@ Status CloudBaseCompaction::modify_rowsets() {
387387
// the tablet to be unable to synchronize the rowset meta changes generated by cumu compaction.
388388
cloud_tablet()->set_base_compaction_cnt(stats.base_compaction_cnt());
389389
if (output_rowset_delete_bitmap) {
390-
_tablet->tablet_meta()->delete_bitmap().merge(*output_rowset_delete_bitmap);
390+
_tablet->tablet_meta()->delete_bitmap()->merge(*output_rowset_delete_bitmap);
391391
}
392392
if (stats.cumulative_compaction_cnt() >= cloud_tablet()->cumulative_compaction_cnt()) {
393393
cloud_tablet()->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(),

‎be/src/cloud/cloud_cumulative_compaction.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,7 @@ Status CloudCumulativeCompaction::modify_rowsets() {
366366
cloud_tablet()->set_cumulative_compaction_cnt(stats.cumulative_compaction_cnt());
367367
cloud_tablet()->set_cumulative_layer_point(stats.cumulative_point());
368368
if (output_rowset_delete_bitmap) {
369-
_tablet->tablet_meta()->delete_bitmap().merge(*output_rowset_delete_bitmap);
369+
_tablet->tablet_meta()->delete_bitmap()->merge(*output_rowset_delete_bitmap);
370370
}
371371
if (stats.base_compaction_cnt() >= cloud_tablet()->base_compaction_cnt()) {
372372
cloud_tablet()->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(),
@@ -406,7 +406,7 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() {
406406
rowset->rowset_id().to_string();
407407
DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
408408
DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version};
409-
auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
409+
auto d = _tablet->tablet_meta()->delete_bitmap()->get_agg(
410410
{rowset->rowset_id(), seg_id, pre_max_version});
411411
to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end));
412412
if (d->isEmpty()) {
@@ -430,10 +430,10 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() {
430430
_input_rowsets.back()->end_version());
431431
for (auto it = new_delete_bitmap->delete_bitmap.begin();
432432
it != new_delete_bitmap->delete_bitmap.end(); it++) {
433-
_tablet->tablet_meta()->delete_bitmap().set(it->first, it->second);
433+
_tablet->tablet_meta()->delete_bitmap()->set(it->first, it->second);
434434
}
435-
_tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(),
436-
to_remove_vec);
435+
_tablet->tablet_meta()->delete_bitmap()->add_to_remove_queue(version.to_string(),
436+
to_remove_vec);
437437
DBUG_EXECUTE_IF(
438438
"CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets",
439439
{ static_cast<CloudTablet*>(_tablet.get())->delete_expired_stale_rowsets(); });

‎be/src/cloud/cloud_delete_bitmap_action.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,9 @@ Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpReque
9494
return Status::NotFound("Tablet not found. tablet_id={}", tablet_id);
9595
}
9696

97-
auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
98-
auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
99-
auto size = tablet->tablet_meta()->delete_bitmap().get_size();
97+
auto count = tablet->tablet_meta()->delete_bitmap()->get_delete_bitmap_count();
98+
auto cardinality = tablet->tablet_meta()->delete_bitmap()->cardinality();
99+
auto size = tablet->tablet_meta()->delete_bitmap()->get_size();
100100
LOG(INFO) << "show_local_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count
101101
<< ",cardinality=" << cardinality << ",size=" << size;
102102

@@ -136,9 +136,9 @@ Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest*
136136
LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st;
137137
return st;
138138
}
139-
auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
140-
auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
141-
auto size = tablet->tablet_meta()->delete_bitmap().get_size();
139+
auto count = tablet->tablet_meta()->delete_bitmap()->get_delete_bitmap_count();
140+
auto cardinality = tablet->tablet_meta()->delete_bitmap()->cardinality();
141+
auto size = tablet->tablet_meta()->delete_bitmap()->get_size();
142142
LOG(INFO) << "show_ms_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count
143143
<< ",cardinality=" << cardinality << ",size=" << size;
144144

@@ -179,4 +179,4 @@ void CloudDeleteBitmapAction::handle(HttpRequest* req) {
179179
}
180180

181181
#include "common/compile_check_end.h"
182-
} // namespace doris
182+
} // namespace doris

‎be/src/cloud/cloud_full_compaction.cpp‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "common/status.h"
2727
#include "cpp/sync_point.h"
2828
#include "gen_cpp/cloud.pb.h"
29+
#include "olap/base_tablet.h"
2930
#include "olap/compaction.h"
3031
#include "olap/rowset/beta_rowset.h"
3132
#include "olap/tablet_meta.h"
@@ -254,7 +255,7 @@ Status CloudFullCompaction::modify_rowsets() {
254255
cloud_tablet()->set_base_compaction_cnt(stats.base_compaction_cnt());
255256
cloud_tablet()->set_cumulative_layer_point(stats.cumulative_point());
256257
if (output_rowset_delete_bitmap) {
257-
_tablet->tablet_meta()->delete_bitmap().merge(*output_rowset_delete_bitmap);
258+
_tablet->tablet_meta()->delete_bitmap()->merge(*output_rowset_delete_bitmap);
258259
}
259260
if (stats.cumulative_compaction_cnt() >= cloud_tablet()->cumulative_compaction_cnt()) {
260261
cloud_tablet()->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(),
@@ -321,8 +322,9 @@ Status CloudFullCompaction::_cloud_full_compaction_update_delete_bitmap(int64_t
321322
int64_t max_version = cloud_tablet()->max_version().second;
322323
DCHECK(max_version >= _output_rowset->version().second);
323324
if (max_version > _output_rowset->version().second) {
324-
RETURN_IF_ERROR(cloud_tablet()->capture_consistent_rowsets_unlocked(
325-
{_output_rowset->version().second + 1, max_version}, &tmp_rowsets));
325+
auto ret = DORIS_TRY(cloud_tablet()->capture_consistent_rowsets_unlocked(
326+
{_output_rowset->version().second + 1, max_version}, CaptureRowsetOps {}));
327+
tmp_rowsets = std::move(ret.rowsets);
326328
}
327329
for (const auto& it : tmp_rowsets) {
328330
int64_t cur_version = it->rowset_meta()->start_version();
@@ -354,7 +356,7 @@ Status CloudFullCompaction::_cloud_full_compaction_update_delete_bitmap(int64_t
354356
.tag("input_segments", _input_segments)
355357
.tag("input_rowsets_total_size", _input_rowsets_total_size)
356358
.tag("update_bitmap_size", delete_bitmap->delete_bitmap.size());
357-
_tablet->tablet_meta()->delete_bitmap().merge(*delete_bitmap);
359+
_tablet->tablet_meta()->delete_bitmap()->merge(*delete_bitmap);
358360
return Status::OK();
359361
}
360362

‎be/src/cloud/cloud_meta_mgr.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -552,7 +552,7 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet* tablet, bool warmup_delta_
552552
.error(st);
553553
return st;
554554
}
555-
tablet->tablet_meta()->delete_bitmap().merge(delete_bitmap);
555+
tablet->tablet_meta()->delete_bitmap()->merge(delete_bitmap);
556556
}
557557
{
558558
const auto& stats = resp.stats();

‎be/src/cloud/cloud_schema_change_job.cpp‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "cloud/cloud_meta_mgr.h"
2828
#include "cloud/cloud_tablet_mgr.h"
2929
#include "common/status.h"
30+
#include "olap/base_tablet.h"
3031
#include "olap/delete_handler.h"
3132
#include "olap/olap_define.h"
3233
#include "olap/rowset/beta_rowset.h"
@@ -168,7 +169,7 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque
168169
reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx();
169170
reader_context.is_unique = _base_tablet->keys_type() == UNIQUE_KEYS;
170171
reader_context.batch_size = ALTER_TABLE_BATCH_SIZE;
171-
reader_context.delete_bitmap = &_base_tablet->tablet_meta()->delete_bitmap();
172+
reader_context.delete_bitmap = _base_tablet->tablet_meta()->delete_bitmap();
172173
reader_context.version = Version(0, start_resp.alter_version());
173174

174175
for (auto& split : rs_splits) {
@@ -436,7 +437,7 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
436437
.tag("alter_version", alter_version);
437438
RETURN_IF_ERROR(_cloud_storage_engine.register_compaction_stop_token(_new_tablet, initiator));
438439
TabletMetaSharedPtr tmp_meta = std::make_shared<TabletMeta>(*(_new_tablet->tablet_meta()));
439-
tmp_meta->delete_bitmap().delete_bitmap.clear();
440+
tmp_meta->delete_bitmap()->delete_bitmap.clear();
440441
std::shared_ptr<CloudTablet> tmp_tablet =
441442
std::make_shared<CloudTablet>(_cloud_storage_engine, tmp_meta);
442443
{
@@ -445,22 +446,21 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
445446
}
446447

447448
// step 1, process incremental rowset without delete bitmap update lock
448-
std::vector<RowsetSharedPtr> incremental_rowsets;
449449
RETURN_IF_ERROR(_cloud_storage_engine.meta_mgr().sync_tablet_rowsets(tmp_tablet.get()));
450450
int64_t max_version = tmp_tablet->max_version().second;
451451
LOG(INFO) << "alter table for mow table, calculate delete bitmap of "
452452
<< "incremental rowsets without lock, version: " << start_calc_delete_bitmap_version
453453
<< "-" << max_version << " new_table_id: " << _new_tablet->tablet_id();
454454
if (max_version >= start_calc_delete_bitmap_version) {
455-
RETURN_IF_ERROR(tmp_tablet->capture_consistent_rowsets_unlocked(
456-
{start_calc_delete_bitmap_version, max_version}, &incremental_rowsets));
455+
auto ret = DORIS_TRY(tmp_tablet->capture_consistent_rowsets_unlocked(
456+
{start_calc_delete_bitmap_version, max_version}, CaptureRowsetOps {}));
457457
DBUG_EXECUTE_IF("CloudSchemaChangeJob::_process_delete_bitmap.after.capture_without_lock",
458458
DBUG_BLOCK);
459459
{
460460
std::unique_lock wlock(tmp_tablet->get_header_lock());
461461
tmp_tablet->add_rowsets(_output_rowsets, true, wlock);
462462
}
463-
for (auto rowset : incremental_rowsets) {
463+
for (auto rowset : ret.rowsets) {
464464
RETURN_IF_ERROR(CloudTablet::update_delete_bitmap_without_lock(tmp_tablet, rowset));
465465
}
466466
}
@@ -476,15 +476,14 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
476476
LOG(INFO) << "alter table for mow table, calculate delete bitmap of "
477477
<< "incremental rowsets with lock, version: " << max_version + 1 << "-"
478478
<< new_max_version << " new_tablet_id: " << _new_tablet->tablet_id();
479-
std::vector<RowsetSharedPtr> new_incremental_rowsets;
480479
if (new_max_version > max_version) {
481-
RETURN_IF_ERROR(tmp_tablet->capture_consistent_rowsets_unlocked(
482-
{max_version + 1, new_max_version}, &new_incremental_rowsets));
480+
auto ret = DORIS_TRY(tmp_tablet->capture_consistent_rowsets_unlocked(
481+
{max_version + 1, new_max_version}, CaptureRowsetOps {}));
483482
{
484483
std::unique_lock wlock(tmp_tablet->get_header_lock());
485484
tmp_tablet->add_rowsets(_output_rowsets, true, wlock);
486485
}
487-
for (auto rowset : new_incremental_rowsets) {
486+
for (auto rowset : ret.rowsets) {
488487
RETURN_IF_ERROR(CloudTablet::update_delete_bitmap_without_lock(tmp_tablet, rowset));
489488
}
490489
}
@@ -501,13 +500,14 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
501500
}
502501
});
503502

504-
auto& delete_bitmap = tmp_tablet->tablet_meta()->delete_bitmap();
503+
auto delete_bitmap = tmp_tablet->tablet_meta()->delete_bitmap();
505504

506505
// step4, store delete bitmap
507506
RETURN_IF_ERROR(_cloud_storage_engine.meta_mgr().update_delete_bitmap(
508-
*_new_tablet, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, initiator, &delete_bitmap));
507+
*_new_tablet, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, initiator, delete_bitmap.get()));
509508

510-
_new_tablet->tablet_meta()->delete_bitmap() = delete_bitmap;
509+
auto original_dbm = _new_tablet->tablet_meta()->delete_bitmap();
510+
*original_dbm = std::move(*delete_bitmap);
511511
return Status::OK();
512512
}
513513

‎be/src/cloud/cloud_tablet.cpp‎

Lines changed: 7 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
#include "common/logging.h"
3838
#include "io/cache/block_file_cache_downloader.h"
3939
#include "io/cache/block_file_cache_factory.h"
40+
#include "olap/base_tablet.h"
4041
#include "olap/compaction.h"
4142
#include "olap/cumulative_compaction_time_series_policy.h"
4243
#include "olap/olap_define.h"
@@ -69,49 +70,17 @@ bool CloudTablet::exceed_version_limit(int32_t limit) {
6970
return _approximate_num_rowsets.load(std::memory_order_relaxed) > limit;
7071
}
7172

72-
Status CloudTablet::capture_consistent_rowsets_unlocked(
73-
const Version& spec_version, std::vector<RowsetSharedPtr>* rowsets) const {
74-
Versions version_path;
75-
auto st = _timestamped_version_tracker.capture_consistent_versions(spec_version, &version_path);
76-
if (!st.ok()) {
77-
// Check no missed versions or req version is merged
78-
auto missed_versions = get_missed_versions(spec_version.second);
79-
if (missed_versions.empty()) {
80-
st.set_code(VERSION_ALREADY_MERGED); // Reset error code
81-
}
82-
st.append(" tablet_id=" + std::to_string(tablet_id()));
83-
return st;
84-
}
85-
VLOG_DEBUG << "capture consitent versions: " << version_path;
86-
return _capture_consistent_rowsets_unlocked(version_path, rowsets);
87-
}
88-
8973
Status CloudTablet::capture_rs_readers(const Version& spec_version,
9074
std::vector<RowSetSplits>* rs_splits,
9175
bool skip_missing_version) {
9276
DBUG_EXECUTE_IF("CloudTablet.capture_rs_readers.return.e-230", {
9377
LOG_WARNING("CloudTablet.capture_rs_readers.return e-230").tag("tablet_id", tablet_id());
9478
return Status::Error<false>(-230, "injected error");
9579
});
96-
Versions version_path;
9780
std::shared_lock rlock(_meta_lock);
98-
auto st = _timestamped_version_tracker.capture_consistent_versions(spec_version, &version_path);
99-
if (!st.ok()) {
100-
rlock.unlock(); // avoid logging in lock range
101-
// Check no missed versions or req version is merged
102-
auto missed_versions = get_missed_versions(spec_version.second);
103-
if (missed_versions.empty()) {
104-
st.set_code(VERSION_ALREADY_MERGED); // Reset error code
105-
st.append(" versions are already compacted, ");
106-
}
107-
st.append(" tablet_id=" + std::to_string(tablet_id()));
108-
// clang-format off
109-
LOG(WARNING) << st << '\n' << [this]() { std::string json; get_compaction_status(&json); return json; }();
110-
// clang-format on
111-
return st;
112-
}
113-
VLOG_DEBUG << "capture consitent versions: " << version_path;
114-
return capture_rs_readers_unlocked(version_path, rs_splits);
81+
*rs_splits = DORIS_TRY(capture_rs_readers_unlocked(
82+
spec_version, CaptureRowsetOps {.skip_missing_versions = skip_missing_version}));
83+
return Status::OK();
11584
}
11685

11786
Status CloudTablet::merge_rowsets_schema() {
@@ -452,7 +421,7 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() {
452421
}
453422
_reconstruct_version_tracker_if_necessary();
454423
}
455-
_tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete);
424+
_tablet_meta->delete_bitmap()->remove_stale_delete_bitmap_from_queue(version_to_delete);
456425
recycle_cached_data(expired_rowsets);
457426
add_unused_rowsets(expired_rowsets);
458427
return expired_rowsets.size();
@@ -882,7 +851,7 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
882851
std::size_t missed_rows_size = 0;
883852
calc_compaction_output_rowset_delete_bitmap(
884853
input_rowsets, rowid_conversion, 0, version.second + 1, missed_rows.get(),
885-
location_map.get(), tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get());
854+
location_map.get(), *tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get());
886855
if (missed_rows) {
887856
missed_rows_size = missed_rows->size();
888857
if (!allow_delete_in_cumu_compaction) {
@@ -917,7 +886,7 @@ Status CloudTablet::calc_delete_bitmap_for_compaction(
917886

918887
calc_compaction_output_rowset_delete_bitmap(
919888
input_rowsets, rowid_conversion, version.second, UINT64_MAX, missed_rows.get(),
920-
location_map.get(), tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get());
889+
location_map.get(), *tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get());
921890
int64_t t4 = MonotonicMicros();
922891
if (location_map) {
923892
RETURN_IF_ERROR(check_rowid_conversion(output_rowset, *location_map));

‎be/src/cloud/cloud_tablet.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,6 @@ class CloudTablet final : public BaseTablet {
5151
Status capture_rs_readers(const Version& spec_version, std::vector<RowSetSplits>* rs_splits,
5252
bool skip_missing_version) override;
5353

54-
Status capture_consistent_rowsets_unlocked(
55-
const Version& spec_version, std::vector<RowsetSharedPtr>* rowsets) const override;
56-
5754
size_t tablet_footprint() override {
5855
return _approximate_data_size.load(std::memory_order_relaxed);
5956
}

‎be/src/cloud/cloud_tablet_mgr.cpp‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include "cloud/cloud_storage_engine.h"
2424
#include "cloud/cloud_tablet.h"
2525
#include "cloud/config.h"
26+
#include "common/logging.h"
2627
#include "common/status.h"
2728
#include "olap/lru_cache.h"
2829
#include "runtime/memory/cache_policy.h"
@@ -151,7 +152,8 @@ void set_tablet_access_time_ms(CloudTablet* tablet) {
151152

152153
Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_id, bool warmup_data,
153154
bool sync_delete_bitmap,
154-
SyncRowsetStats* sync_stats) {
155+
SyncRowsetStats* sync_stats,
156+
bool local_only) {
155157
// LRU value type. `Value`'s lifetime MUST NOT be longer than `CloudTabletMgr`
156158
class Value : public LRUCacheValueBase {
157159
public:
@@ -169,6 +171,16 @@ Result<std::shared_ptr<CloudTablet>> CloudTabletMgr::get_tablet(int64_t tablet_i
169171
CacheKey key(tablet_id_str);
170172
auto* handle = _cache->lookup(key);
171173
if (handle == nullptr) {
174+
if (local_only) {
175+
LOG(INFO) << "tablet=" << tablet_id
176+
<< "does not exists in local tablet cache, because param local_only=true, "
177+
"treat it as an error";
178+
return ResultError(Status::InternalError(
179+
"tablet={} does not exists in local tablet cache, because param "
180+
"local_only=true, "
181+
"treat it as an error",
182+
tablet_id));
183+
}
172184
auto load_tablet = [this, &key, warmup_data, sync_delete_bitmap,
173185
sync_stats](int64_t tablet_id) -> std::shared_ptr<CloudTablet> {
174186
TabletMetaSharedPtr tablet_meta;
@@ -451,7 +463,7 @@ void CloudTabletMgr::get_topn_tablet_delete_bitmap_score(
451463
auto t = tablet_wk.lock();
452464
if (!t) return;
453465
uint64_t delete_bitmap_count =
454-
t.get()->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
466+
t.get()->tablet_meta()->delete_bitmap()->get_delete_bitmap_count();
455467
total_delete_map_count += delete_bitmap_count;
456468
if (delete_bitmap_count > *max_delete_bitmap_score) {
457469
max_delete_bitmap_score_tablet_id = t->tablet_id();

‎be/src/cloud/cloud_tablet_mgr.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ class CloudTabletMgr {
4646
// sync rowsets after, and download segment data in background if `warmup_data` is true.
4747
Result<std::shared_ptr<CloudTablet>> get_tablet(int64_t tablet_id, bool warmup_data = false,
4848
bool sync_delete_bitmap = true,
49-
SyncRowsetStats* sync_stats = nullptr);
49+
SyncRowsetStats* sync_stats = nullptr,
50+
bool local_only = false);
5051

5152
void erase_tablet(int64_t tablet_id);
5253

0 commit comments

Comments
 (0)