From 1d77e14e74072f8fe31940a111e81ab0eace30fe Mon Sep 17 00:00:00 2001 From: Slavasil Date: Thu, 12 Dec 2024 16:17:00 +0300 Subject: [PATCH] track post dates instead of ids --- manager.cpp | 57 ++++++++++++++++++++++++++++++----------------------- state.cpp | 28 +++++++++++++------------- state.h | 4 ++-- 3 files changed, 48 insertions(+), 41 deletions(-) diff --git a/manager.cpp b/manager.cpp index a743d18..33bc41f 100644 --- a/manager.cpp +++ b/manager.cpp @@ -120,10 +120,13 @@ void NewPostFetcher::fetch() { vkReady = false; if (state.needRequest) { auto &wall = mgr->m_appConfig->vkSources[state.sourceIndex].id; - if (mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostId != 0) { + if (mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostDate != 0) { spdlog::info("[vk:{}] fetching {} VK posts at offset {}", i, state.count, state.offset); state.needRequest = false; mgr->collect_vk_posts_from(wall, state.offset, state.count, [&, i](auto posts) mutable { + for (auto &p : posts) { + spdlog::debug("[vk:{}] got post dated {}", i, p.date); + } state.offset += posts.size(); state.count = state.count * 3 / 2; check_vk_posts(i, posts); @@ -134,8 +137,8 @@ void NewPostFetcher::fetch() { mgr->collect_all_vk_posts(wall, [&, i](auto posts) mutable { spdlog::info("[vk:{}] fetched all {} posts", i, posts.size()); if (posts.size() > 0) { - spdlog::info("[vk:{}] last loaded post id is now {}", i, posts[0].id); - mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostId = posts[0].id; + spdlog::info("[vk:{}] last loaded post date is now {}", i, posts[0].date); + mgr->m_appState->vkRepostState[state.sourceIndex].lastLoadedPostDate = posts[0].date; } state.ready = true; std::vector aposts = mgr->to_abstract_posts(posts, state.sourceIndex); @@ -156,11 +159,15 @@ void NewPostFetcher::fetch() { tgReady = false; if (state.needRequest) { long channel = mgr->m_appConfig->tgSources[state.sourceIndex].id; - if (mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostId != 0) { + if (mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostDate != 0) { spdlog::info("[tg:{}] fetching {} posts starting from #{}", i, state.count, state.offset); state.needRequest = false; - mgr->collect_tg_posts_from(channel, state.offset, state.count, [this, i, state](auto posts) mutable { + mgr->collect_tg_posts_from(channel, state.offset, state.count, [this, i, &state](auto posts) mutable { + for (auto &p : posts) { + spdlog::debug("[tg:{}] got post dated {}", i, p->date_); + } state.offset = posts[posts.size() - 1]->id_; + spdlog::info("[tg:{}] setting from to id {}", i, posts[posts.size() - 1]->id_); check_tg_posts(i, std::move(posts)); }); state.count = state.count * 3 / 2; @@ -169,8 +176,8 @@ void NewPostFetcher::fetch() { mgr->collect_all_tg_posts(channel, [this, i, &state](auto posts) mutable { spdlog::info("[tg:{}] fetched all {} posts", i, posts.size()); if (posts.size() > 0) { - spdlog::info("[tg:{}] last loaded post id is now {}", i, posts[0]->id_); - mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostId = posts[0]->id_; + spdlog::info("[tg:{}] last loaded post date is now {}", i, posts[0]->date_); + mgr->m_appState->tgRepostState[state.sourceIndex].lastLoadedPostDate = posts[0]->date_; } state.ready = true; std::vector aposts = mgr->to_abstract_posts(posts, state.sourceIndex); @@ -241,13 +248,13 @@ void NewPostFetcher::check_vk_posts(int index, std::vector posts) { spdlog::info("[vk:{}] fetched {} posts", index, posts.size()); auto &state = vkState[index]; auto &appState = mgr->m_appState->vkRepostState[state.sourceIndex]; - long oldLastPostId = appState.lastLoadedPostId; + long oldLastPostDate = appState.lastLoadedPostDate; if (posts.size() > 0) { - spdlog::info("[vk:{}] last post id is now {}", index, posts[0].id); + spdlog::info("[vk:{}] last post date is now {}", index, posts[0].date); } std::vector aposts = mgr->to_abstract_posts(posts, state.sourceIndex); - spdlog::info("[vk:{}] looking for id {}, have {} - {}", index, oldLastPostId, aposts[0].id, aposts[aposts.size() - 1].id); - if (mgr->drop_posts_older_than(aposts, oldLastPostId)) { + spdlog::info("[vk:{}] looking for date {}, have {} - {}", index, oldLastPostDate, aposts[0].id, aposts[aposts.size() - 1].id); + if (mgr->drop_posts_older_than(aposts, oldLastPostDate)) { spdlog::info("[vk:{}] found last remembered post", index); state.ready = true; } @@ -257,8 +264,8 @@ void NewPostFetcher::check_vk_posts(int index, std::vector posts) { } state.needRequest = true; if (state.ready && !state.posts.empty()) { - spdlog::debug("[vk:{}] last loaded post id is now {}", index, state.posts[0].id); - appState.lastLoadedPostId = state.posts[0].id; + spdlog::debug("[vk:{}] last loaded post date is now {}", index, state.posts[0].date); + appState.lastLoadedPostDate = state.posts[0].id; } fetch(); } @@ -267,12 +274,12 @@ void NewPostFetcher::check_tg_posts(int index, std::vectorm_appState->tgRepostState[state.sourceIndex]; - long oldLastPostId = appState.lastLoadedPostId; + long oldLastPostDate = appState.lastLoadedPostDate; if (posts.size() > 0) { - spdlog::info("[tg:{}] last post id is now {}", index, posts[0]->id_); + spdlog::info("[tg:{}] last post date is now {}", index, posts[0]->date_); } std::vector aposts = mgr->to_abstract_posts(posts, state.sourceIndex); - if (mgr->drop_posts_older_than(aposts, oldLastPostId)) { + if (mgr->drop_posts_older_than(aposts, oldLastPostDate)) { spdlog::info("[tg:{}] found last remembered post", index); state.ready = true; } @@ -282,18 +289,18 @@ void NewPostFetcher::check_tg_posts(int index, std::vectorvkRepostState) { - appState.lastLoadedPostId = appState.lastForwardedPostId; + appState.lastLoadedPostDate = appState.lastForwardedPostDate; } for (auto &appState : m_appState->tgRepostState) { - appState.lastLoadedPostId = appState.lastForwardedPostId; + appState.lastLoadedPostDate = appState.lastForwardedPostDate; } NewPostFetcher *f = new NewPostFetcher(this, true, true); @@ -403,8 +410,8 @@ void RepostManager::collect_tg_posts_from__intermediate(long channel, long from, }); } -bool RepostManager::drop_posts_older_than(std::vector &posts, long lastPostId) { - auto idx = std::find_if(posts.begin(), posts.end(), [lastPostId](auto &post){ return post.id == lastPostId; }); +bool RepostManager::drop_posts_older_than(std::vector &posts, long lastPostDate) { + auto idx = std::find_if(posts.begin(), posts.end(), [lastPostDate](auto &post){ return post.date <= lastPostDate; }); if (idx == posts.end()) { return false; } else { @@ -493,16 +500,16 @@ void RepostManager::repost(AbstractPost &post) { //entities.push_back(std::move(td_api::make_object(signatureStart, signatureLength, td_api::make_object()))); auto content = td_api::make_object(td_api::make_object(post.text, std::move(entities)), td_api::make_object(true, std::string(""), false, false, false), false); spdlog::info("reposting to {}", m_appConfig->tgDestinationId); - m_tg.send_query(td_api::make_object(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), [this, postId = post.id, srcType = post.sourceType, src = post.source](auto result){ + m_tg.send_query(td_api::make_object(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), [this, postDate = post.date, srcType = post.sourceType, src = post.source](auto result){ if (result->get_id() == td_api::error::ID) { auto &err = (td_api::error&)*result; spdlog::error("sendMessage error: {} {}", err.code_, err.message_); uv_timer_stop(m_repostTimer); } else { if (srcType == posts::SRC_VK) - m_appState->vkRepostState[src].lastForwardedPostId = postId; + m_appState->vkRepostState[src].lastForwardedPostDate = postDate; else - m_appState->tgRepostState[src].lastForwardedPostId = postId; + m_appState->tgRepostState[src].lastForwardedPostDate = postDate; } }); } diff --git a/state.cpp b/state.cpp index e5c801c..09cd717 100644 --- a/state.cpp +++ b/state.cpp @@ -55,13 +55,13 @@ AppState::AppState(std::string filename, config::AppConfig *cfg) : m_saveFilenam ) ); } - if (vkState_.contains("last_post_id")) { - json lastPostId_ = vkState_["last_post_id"]; - if (lastPostId_.type() == json::value_t::number_integer || - lastPostId_.type() == json::value_t::number_unsigned) { - s.lastForwardedPostId = vkState_["last_post_id"]; + if (vkState_.contains("last_post_date")) { + json lastPostDate_ = vkState_["last_post_date"]; + if (lastPostDate_.type() == json::value_t::number_integer || + lastPostDate_.type() == json::value_t::number_unsigned) { + s.lastForwardedPostDate = vkState_["last_post_date"]; } else { - throw InvalidSavedStateException("key vk.last_post_id must be an integer"); + throw InvalidSavedStateException("key vk.last_post_date must be an integer"); } } if (sourceIndex < m_config->vkSources.size()) @@ -91,13 +91,13 @@ AppState::AppState(std::string filename, config::AppConfig *cfg) : m_saveFilenam [=](auto &src){ return src.id == keyNum; } ) ); - if (tgState_.contains("last_post_id")) { - json lastPostId_ = tgState_["last_post_id"]; - if (lastPostId_.type() == json::value_t::number_integer || - lastPostId_.type() == json::value_t::number_unsigned) { - s.lastForwardedPostId = tgState_["last_post_id"]; + if (tgState_.contains("last_post_date")) { + json lastPostDate_ = tgState_["last_post_date"]; + if (lastPostDate_.type() == json::value_t::number_integer || + lastPostDate_.type() == json::value_t::number_unsigned) { + s.lastForwardedPostDate = tgState_["last_post_date"]; } else { - throw InvalidSavedStateException("key tg.last_post_id must be an integer"); + throw InvalidSavedStateException("key tg.last_post_date must be an integer"); } } if (sourceIndex < m_config->tgSources.size()) @@ -115,7 +115,7 @@ void AppState::save() { json state = {{"vk", json::object()}, {"tg", json::object()}}; for (int i = 0; i < m_config->vkSources.size(); ++i) { auto &s = vkRepostState[i]; - json st = {{"last_post_id", s.lastForwardedPostId}}; + json st = {{"last_post_date", s.lastForwardedPostDate}}; auto &vkId = m_config->vkSources[i].id; std::string vkIdStr; if (vkId.index() == 0) vkIdStr = std::to_string(std::get(vkId)); @@ -124,7 +124,7 @@ void AppState::save() { } for (int i = 0; i < m_config->tgSources.size(); ++i) { auto &s = tgRepostState[i]; - json st = {{"last_post_id", s.lastForwardedPostId}}; + json st = {{"last_post_date", s.lastForwardedPostDate}}; state["tg"].emplace(std::to_string(m_config->tgSources[i].id), st); } std::ofstream f(m_saveFilename.c_str()); diff --git a/state.h b/state.h index b49b123..cb03408 100644 --- a/state.h +++ b/state.h @@ -13,8 +13,8 @@ namespace state { }; struct RepostState { - long lastLoadedPostId = 0; - long lastForwardedPostId = 0; + long lastLoadedPostDate = 0; + long lastForwardedPostDate = 0; }; class AppState {