Slavasil/mmcs-quotes-bridge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

track post dates instead of ids

Browse Source
This commit is contained in:
Slavasil 2024-12-12 16:17:00 +03:00
parent b95b500873
commit 1d77e14e74
3 changed files with 48 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
manager.cpp
View File

@ -120,10 +120,13 @@ void NewPostFetcher::fetch() {
vkReady = false; vkReady = false;
if (state.needRequest) { if (state.needRequest) {
auto &wall = mgr->m_appConfig->vkSources[state.sourceIndex].id; auto &wall = mgr->m_appConfig->vkSources[state.sourceIndex].id;
if (mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostId != 0) { if (mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostDate != 0) {
spdlog::info("[vk:{}] fetching {} VK posts at offset {}", i, state.count, state.offset); spdlog::info("[vk:{}] fetching {} VK posts at offset {}", i, state.count, state.offset);
state.needRequest = false; state.needRequest = false;
mgr->collect_vk_posts_from(wall, state.offset, state.count, [&, i](auto posts) mutable { mgr->collect_vk_posts_from(wall, state.offset, state.count, [&, i](auto posts) mutable {
for (auto &p : posts) {
spdlog::debug("[vk:{}] got post dated {}", i, p.date);
}
state.offset += posts.size(); state.offset += posts.size();
state.count = state.count * 3 / 2; state.count = state.count * 3 / 2;
check_vk_posts(i, posts); check_vk_posts(i, posts);
@ -134,8 +137,8 @@ void NewPostFetcher::fetch() {
mgr->collect_all_vk_posts(wall, [&, i](auto posts) mutable { mgr->collect_all_vk_posts(wall, [&, i](auto posts) mutable {
spdlog::info("[vk:{}] fetched all {} posts", i, posts.size()); spdlog::info("[vk:{}] fetched all {} posts", i, posts.size());
if (posts.size() > 0) { if (posts.size() > 0) {
spdlog::info("[vk:{}] last loaded post id is now {}", i, posts[0].id); spdlog::info("[vk:{}] last loaded post date is now {}", i, posts[0].date);
mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostId = posts[0].id; mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostDate = posts[0].date;
} }
state.ready = true; state.ready = true;
std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex); std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex);
@ -156,11 +159,15 @@ void NewPostFetcher::fetch() {
tgReady = false; tgReady = false;
if (state.needRequest) { if (state.needRequest) {
long channel = mgr->m_appConfig->tgSources[state.sourceIndex].id; long channel = mgr->m_appConfig->tgSources[state.sourceIndex].id;
if (mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostId != 0) { if (mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostDate != 0) {
spdlog::info("[tg:{}] fetching {} posts starting from #{}", i, state.count, state.offset); spdlog::info("[tg:{}] fetching {} posts starting from #{}", i, state.count, state.offset);
state.needRequest = false; state.needRequest = false;
mgr->collect_tg_posts_from(channel, state.offset, state.count, [this, i, state](auto posts) mutable { mgr->collect_tg_posts_from(channel, state.offset, state.count, [this, i, &state](auto posts) mutable {
for (auto &p : posts) {
spdlog::debug("[tg:{}] got post dated {}", i, p->date_);
}
state.offset = posts[posts.size() - 1]->id_; state.offset = posts[posts.size() - 1]->id_;
spdlog::info("[tg:{}] setting from to id {}", i, posts[posts.size() - 1]->id_);
check_tg_posts(i, std::move(posts)); check_tg_posts(i, std::move(posts));
}); });
state.count = state.count * 3 / 2; state.count = state.count * 3 / 2;
@ -169,8 +176,8 @@ void NewPostFetcher::fetch() {
mgr->collect_all_tg_posts(channel, [this, i, &state](auto posts) mutable { mgr->collect_all_tg_posts(channel, [this, i, &state](auto posts) mutable {
spdlog::info("[tg:{}] fetched all {} posts", i, posts.size()); spdlog::info("[tg:{}] fetched all {} posts", i, posts.size());
if (posts.size() > 0) { if (posts.size() > 0) {
spdlog::info("[tg:{}] last loaded post id is now {}", i, posts[0]->id_); spdlog::info("[tg:{}] last loaded post date is now {}", i, posts[0]->date_);
mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostId = posts[0]->id_; mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostDate = posts[0]->date_;
} }
state.ready = true; state.ready = true;
std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex); std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex);
@ -241,13 +248,13 @@ void NewPostFetcher::check_vk_posts(int index, std::vector<vk::Post> posts) {
spdlog::info("[vk:{}] fetched {} posts", index, posts.size()); spdlog::info("[vk:{}] fetched {} posts", index, posts.size());
auto &state = vkState[index]; auto &state = vkState[index];
auto &appState = mgr->m_appState->vkRepostState[state.sourceIndex]; auto &appState = mgr->m_appState->vkRepostState[state.sourceIndex];
long oldLastPostId = appState.lastLoadedPostId; long oldLastPostDate = appState.lastLoadedPostDate;
if (posts.size() > 0) { if (posts.size() > 0) {
spdlog::info("[vk:{}] last post id is now {}", index, posts[0].id); spdlog::info("[vk:{}] last post date is now {}", index, posts[0].date);
} }
std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex); std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex);
spdlog::info("[vk:{}] looking for id {}, have {} - {}", index, oldLastPostId, aposts[0].id, aposts[aposts.size() - 1].id); spdlog::info("[vk:{}] looking for date {}, have {} - {}", index, oldLastPostDate, aposts[0].id, aposts[aposts.size() - 1].id);
if (mgr->drop_posts_older_than(aposts, oldLastPostId)) { if (mgr->drop_posts_older_than(aposts, oldLastPostDate)) {
spdlog::info("[vk:{}] found last remembered post", index); spdlog::info("[vk:{}] found last remembered post", index);
state.ready = true; state.ready = true;
} }
@ -257,8 +264,8 @@ void NewPostFetcher::check_vk_posts(int index, std::vector<vk::Post> posts) {
} }
state.needRequest = true; state.needRequest = true;
if (state.ready && !state.posts.empty()) { if (state.ready && !state.posts.empty()) {
spdlog::debug("[vk:{}] last loaded post id is now {}", index, state.posts[0].id); spdlog::debug("[vk:{}] last loaded post date is now {}", index, state.posts[0].date);
appState.lastLoadedPostId = state.posts[0].id; appState.lastLoadedPostDate = state.posts[0].id;
} }
fetch(); fetch();
} }
@ -267,12 +274,12 @@ void NewPostFetcher::check_tg_posts(int index, std::vector<td::tl::unique_ptr<td
spdlog::info("[tg:{}] fetched {} posts", index, posts.size()); spdlog::info("[tg:{}] fetched {} posts", index, posts.size());
auto &state = tgState[index]; auto &state = tgState[index];
auto &appState = mgr->m_appState->tgRepostState[state.sourceIndex]; auto &appState = mgr->m_appState->tgRepostState[state.sourceIndex];
long oldLastPostId = appState.lastLoadedPostId; long oldLastPostDate = appState.lastLoadedPostDate;
if (posts.size() > 0) { if (posts.size() > 0) {
spdlog::info("[tg:{}] last post id is now {}", index, posts[0]->id_); spdlog::info("[tg:{}] last post date is now {}", index, posts[0]->date_);
} }
std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex); std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts, state.sourceIndex);
if (mgr->drop_posts_older_than(aposts, oldLastPostId)) { if (mgr->drop_posts_older_than(aposts, oldLastPostDate)) {
spdlog::info("[tg:{}] found last remembered post", index); spdlog::info("[tg:{}] found last remembered post", index);
state.ready = true; state.ready = true;
} }
@ -282,18 +289,18 @@ void NewPostFetcher::check_tg_posts(int index, std::vector<td::tl::unique_ptr<td
} }
state.needRequest = true; state.needRequest = true;
if (state.ready && !state.posts.empty()) { if (state.ready && !state.posts.empty()) {
spdlog::debug("[tg:{}] last loaded post id is now {}", index, state.posts[0].id); spdlog::debug("[tg:{}] last loaded post date is now {}", index, state.posts[0].date);
appState.lastLoadedPostId = state.posts[0].id; appState.lastLoadedPostDate = state.posts[0].id;
} }
fetch(); fetch();
} }
void RepostManager::on_clients_ready() { void RepostManager::on_clients_ready() {
for (auto &appState : m_appState->vkRepostState) { for (auto &appState : m_appState->vkRepostState) {
appState.lastLoadedPostId = appState.lastForwardedPostId; appState.lastLoadedPostDate = appState.lastForwardedPostDate;
} }
for (auto &appState : m_appState->tgRepostState) { for (auto &appState : m_appState->tgRepostState) {
appState.lastLoadedPostId = appState.lastForwardedPostId; appState.lastLoadedPostDate = appState.lastForwardedPostDate;
} }
NewPostFetcher *f = new NewPostFetcher(this, true, true); NewPostFetcher *f = new NewPostFetcher(this, true, true);
@ -403,8 +410,8 @@ void RepostManager::collect_tg_posts_from__intermediate(long channel, long from,
}); });
} }
bool RepostManager::drop_posts_older_than(std::vector<AbstractPost> &posts, long lastPostId) { bool RepostManager::drop_posts_older_than(std::vector<AbstractPost> &posts, long lastPostDate) {
auto idx = std::find_if(posts.begin(), posts.end(), [lastPostId](auto &post){ return post.id == lastPostId; }); auto idx = std::find_if(posts.begin(), posts.end(), [lastPostDate](auto &post){ return post.date <= lastPostDate; });
if (idx == posts.end()) { if (idx == posts.end()) {
return false; return false;
} else { } else {
@ -493,16 +500,16 @@ void RepostManager::repost(AbstractPost &post) {
//entities.push_back(std::move(td_api::make_object<td_api::textEntity>(signatureStart, signatureLength, td_api::make_object<td_api::textEntityTypeSpoiler>()))); //entities.push_back(std::move(td_api::make_object<td_api::textEntity>(signatureStart, signatureLength, td_api::make_object<td_api::textEntityTypeSpoiler>())));
auto content = td_api::make_object<td_api::inputMessageText>(td_api::make_object<td_api::formattedText>(post.text, std::move(entities)), td_api::make_object<td_api::linkPreviewOptions>(true, std::string(""), false, false, false), false); auto content = td_api::make_object<td_api::inputMessageText>(td_api::make_object<td_api::formattedText>(post.text, std::move(entities)), td_api::make_object<td_api::linkPreviewOptions>(true, std::string(""), false, false, false), false);
spdlog::info("reposting to {}", m_appConfig->tgDestinationId); spdlog::info("reposting to {}", m_appConfig->tgDestinationId);
m_tg.send_query(td_api::make_object<td_api::sendMessage>(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), [this, postId = post.id, srcType = post.sourceType, src = post.source](auto result){ m_tg.send_query(td_api::make_object<td_api::sendMessage>(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), [this, postDate = post.date, srcType = post.sourceType, src = post.source](auto result){
if (result->get_id() == td_api::error::ID) { if (result->get_id() == td_api::error::ID) {
auto &err = (td_api::error&)*result; auto &err = (td_api::error&)*result;
spdlog::error("sendMessage error: {} {}", err.code_, err.message_); spdlog::error("sendMessage error: {} {}", err.code_, err.message_);
uv_timer_stop(m_repostTimer); uv_timer_stop(m_repostTimer);
} else { } else {
if (srcType == posts::SRC_VK) if (srcType == posts::SRC_VK)
m_appState->vkRepostState[src].lastForwardedPostId = postId; m_appState->vkRepostState[src].lastForwardedPostDate = postDate;
else else
m_appState->tgRepostState[src].lastForwardedPostId = postId; m_appState->tgRepostState[src].lastForwardedPostDate = postDate;
} }
}); });
} }

28
state.cpp
View File

@ -55,13 +55,13 @@ AppState::AppState(std::string filename, config::AppConfig *cfg) : m_saveFilenam
) )
); );
} }
if (vkState_.contains("last_post_id")) { if (vkState_.contains("last_post_date")) {
json lastPostId_ = vkState_["last_post_id"]; json lastPostDate_ = vkState_["last_post_date"];
if (lastPostId_.type() == json::value_t::number_integer || if (lastPostDate_.type() == json::value_t::number_integer ||
lastPostId_.type() == json::value_t::number_unsigned) { lastPostDate_.type() == json::value_t::number_unsigned) {
s.lastForwardedPostId = vkState_["last_post_id"]; s.lastForwardedPostDate = vkState_["last_post_date"];
} else { } else {
throw InvalidSavedStateException("key vk.last_post_id must be an integer"); throw InvalidSavedStateException("key vk.last_post_date must be an integer");
} }
} }
if (sourceIndex < m_config->vkSources.size()) if (sourceIndex < m_config->vkSources.size())
@ -91,13 +91,13 @@ AppState::AppState(std::string filename, config::AppConfig *cfg) : m_saveFilenam
[=](auto &src){ return src.id == keyNum; } [=](auto &src){ return src.id == keyNum; }
) )
); );
if (tgState_.contains("last_post_id")) { if (tgState_.contains("last_post_date")) {
json lastPostId_ = tgState_["last_post_id"]; json lastPostDate_ = tgState_["last_post_date"];
if (lastPostId_.type() == json::value_t::number_integer || if (lastPostDate_.type() == json::value_t::number_integer ||
lastPostId_.type() == json::value_t::number_unsigned) { lastPostDate_.type() == json::value_t::number_unsigned) {
s.lastForwardedPostId = tgState_["last_post_id"]; s.lastForwardedPostDate = tgState_["last_post_date"];
} else { } else {
throw InvalidSavedStateException("key tg.last_post_id must be an integer"); throw InvalidSavedStateException("key tg.last_post_date must be an integer");
} }
} }
if (sourceIndex < m_config->tgSources.size()) if (sourceIndex < m_config->tgSources.size())
@ -115,7 +115,7 @@ void AppState::save() {
json state = {{"vk", json::object()}, {"tg", json::object()}}; json state = {{"vk", json::object()}, {"tg", json::object()}};
for (int i = 0; i < m_config->vkSources.size(); ++i) { for (int i = 0; i < m_config->vkSources.size(); ++i) {
auto &s = vkRepostState[i]; auto &s = vkRepostState[i];
json st = {{"last_post_id", s.lastForwardedPostId}}; json st = {{"last_post_date", s.lastForwardedPostDate}};
auto &vkId = m_config->vkSources[i].id; auto &vkId = m_config->vkSources[i].id;
std::string vkIdStr; std::string vkIdStr;
if (vkId.index() == 0) vkIdStr = std::to_string(std::get<long>(vkId)); if (vkId.index() == 0) vkIdStr = std::to_string(std::get<long>(vkId));
@ -124,7 +124,7 @@ void AppState::save() {
} }
for (int i = 0; i < m_config->tgSources.size(); ++i) { for (int i = 0; i < m_config->tgSources.size(); ++i) {
auto &s = tgRepostState[i]; auto &s = tgRepostState[i];
json st = {{"last_post_id", s.lastForwardedPostId}}; json st = {{"last_post_date", s.lastForwardedPostDate}};
state["tg"].emplace(std::to_string(m_config->tgSources[i].id), st); state["tg"].emplace(std::to_string(m_config->tgSources[i].id), st);
} }
std::ofstream f(m_saveFilename.c_str()); std::ofstream f(m_saveFilename.c_str());

4
state.h
View File

@ -13,8 +13,8 @@ namespace state {
}; };
struct RepostState { struct RepostState {
long lastLoadedPostId = 0; long lastLoadedPostDate = 0;
long lastForwardedPostId = 0; long lastForwardedPostDate = 0;
}; };
class AppState { class AppState {