diff --git a/config.cpp b/config.cpp index 887264f..546a8af 100644 --- a/config.cpp +++ b/config.cpp @@ -1,5 +1,6 @@ #include "config.h" #include "nlohmann/json.hpp" +#include "spdlog/spdlog.h" #include #include @@ -13,6 +14,8 @@ const char *JSON_KEY_TG_PHONE_NUMBER = "tg_phone_number"; const char *JSON_KEY_VK_SOURCE = "vk_source"; const char *JSON_KEY_TG_SOURCE_ID = "tg_source_id"; const char *JSON_KEY_TG_DESTINATION_ID = "tg_destination_id"; +const char *JSON_KEY_VK_SOURCE_LINK = "vk_source_link"; +const char *JSON_KEY_TG_SOURCE_LINK = "tg_source_link"; const char *ERR_INVALID_TYPE_VK_SERVICE_KEY = "vk_service_key must be a string"; const char *ERR_INVALID_TYPE_TG_API_ID = "tg_api_id must be an integer or a string"; @@ -21,6 +24,8 @@ const char *ERR_INVALID_TYPE_TG_PHONE_NUMBER = "tg_phone_number must be a string const char *ERR_INVALID_TYPE_VK_SOURCE = "vk_source must be an integer or a string"; const char *ERR_INVALID_TYPE_TG_SOURCE_ID = "tg_source_id must be an integer"; const char *ERR_INVALID_TYPE_TG_DESTINATION_ID = "tg_destination_id must be an integer"; +const char *ERR_INVALID_TYPE_VK_SOURCE_LINK = "vk_source_link must be a string"; +const char *ERR_INVALID_TYPE_TG_SOURCE_LINK = "tg_source_link must be a string"; AppConfig::AppConfig(const std::string &filename) { std::ifstream f(filename); @@ -98,4 +103,22 @@ AppConfig::AppConfig(const std::string &filename) { throw InvalidConfigException(ERR_INVALID_TYPE_TG_DESTINATION_ID); } } + + if (config.contains(JSON_KEY_VK_SOURCE_LINK)) { + json vkSourceLink_ = config[JSON_KEY_VK_SOURCE_LINK]; + if (vkSourceLink_.type() == json::value_t::string) { + vkSourceLink = vkSourceLink_; + } else { + spdlog::warn("config: {}", ERR_INVALID_TYPE_VK_SOURCE_LINK); + } + } + + if (config.contains(JSON_KEY_TG_SOURCE_LINK)) { + json tgSourceLink_ = config[JSON_KEY_TG_SOURCE_LINK]; + if (tgSourceLink_.type() == json::value_t::string) { + tgSourceLink = tgSourceLink_; + } else { + spdlog::warn("config: {}", ERR_INVALID_TYPE_TG_SOURCE_LINK); + } + } } \ No newline at end of file diff --git a/config.h b/config.h index 3a245c6..da5c0e6 100644 --- a/config.h +++ b/config.h @@ -18,6 +18,7 @@ namespace config { std::string tgApiHash; std::string tgPhoneNumber; std::variant vkSource; + std::string vkSourceLink, tgSourceLink; long tgSourceId, tgDestinationId; }; } \ No newline at end of file diff --git a/manager.cpp b/manager.cpp index 3600c97..131e094 100644 --- a/manager.cpp +++ b/manager.cpp @@ -1,7 +1,9 @@ #include "manager.h" +#include "posts.h" #include "spdlog/spdlog.h" #include "state.h" #include "td/telegram/td_api.h" +#include "uv.h" #include "vk.h" #include #include @@ -10,6 +12,8 @@ using namespace manager; +const unsigned long REPOST_INTERVAL = 2000; + RepostManager::RepostManager(uv_loop_t *eventLoop, tg::AuthCodeProvider tgCodeProvider, tg::PasswordProvider tgPasswordProvider, state::AppState *appState, config::AppConfig *config) : m_vk(eventLoop), m_tg(eventLoop, config->tgApiId, config->tgApiHash, config->tgPhoneNumber) { m_appState = appState; @@ -17,6 +21,17 @@ RepostManager::RepostManager(uv_loop_t *eventLoop, tg::AuthCodeProvider tgCodePr m_tg.authCodeProvider = tgCodeProvider; m_tg.passwordProvider = tgPasswordProvider; m_vk.set_service_api_key(config->vkServiceKey); + + m_repostTimer = new uv_timer_t; + uv_timer_init(eventLoop, m_repostTimer); + m_repostTimer->data = this; +} + +RepostManager::~RepostManager() { + if (m_repostTimer) { + uv_timer_stop(m_repostTimer); + uv_close((uv_handle_t*)m_repostTimer, [](uv_handle_t *h){ delete h; }); + } } void RepostManager::load_more_telegram_chats() { @@ -52,6 +67,9 @@ void RepostManager::start() { } void RepostManager::on_clients_ready() { + m_appState->vkLastLoadedPostId = m_appState->vkLastPostId; + m_appState->tgLastLoadedPostId = m_appState->tgLastPostId; + struct new_post_fetcher { struct fetcher_state { bool ready = false; @@ -71,12 +89,14 @@ void RepostManager::on_clients_ready() { } if (!vkState.ready && vkState.needRequest) { - if (mgr->m_appState->vkLastPostId != 0) { + if (mgr->m_appState->vkLastLoadedPostId != 0) { spdlog::info("fetching {} VK posts at offset {}", vkState.count, vkState.offset); vkState.needRequest = false; - mgr->collect_vk_posts_from(vkState.offset, vkState.count, [this](auto posts){ check_vk_posts(posts); }); - vkState.offset += vkState.count; - vkState.count = vkState.count * 3 / 2; + mgr->collect_vk_posts_from(vkState.offset, vkState.count, [this](auto posts){ + vkState.offset += posts.size(); + vkState.count = vkState.count * 3 / 2; + check_vk_posts(posts); + }); } else { spdlog::info("fetching all VK posts"); vkState.needRequest = false; @@ -84,7 +104,7 @@ void RepostManager::on_clients_ready() { spdlog::info("fetched all {} VK posts", posts.size()); if (posts.size() > 0) { spdlog::info("last vk post id is now {}", posts[0].id); - mgr->m_appState->vkLastPostId = posts[0].id; + mgr->m_appState->vkLastLoadedPostId = posts[0].id; } vkState.ready = true; std::vector aposts = mgr->to_abstract_posts(posts); @@ -98,7 +118,7 @@ void RepostManager::on_clients_ready() { } if (!tgState.ready && tgState.needRequest) { - if (mgr->m_appState->tgLastPostId != 0) { + if (mgr->m_appState->tgLastLoadedPostId != 0) { spdlog::info("fetching {} TG posts starting from #{}", tgState.count, tgState.offset); tgState.needRequest = false; mgr->collect_tg_posts_from(tgState.offset, tgState.count, [this](auto posts){ @@ -113,7 +133,7 @@ void RepostManager::on_clients_ready() { spdlog::info("fetched all {} TG posts", posts.size()); if (posts.size() > 0) { spdlog::info("last telegram post id is now {}", posts[0]->id_); - mgr->m_appState->tgLastPostId = posts[0]->id_; + mgr->m_appState->tgLastLoadedPostId = posts[0]->id_; } tgState.ready = true; std::vector aposts = mgr->to_abstract_posts(posts); @@ -128,12 +148,13 @@ void RepostManager::on_clients_ready() { } void check_vk_posts(std::vector posts) { spdlog::info("fetched {} VK posts", posts.size()); - long oldLastPostId = mgr->m_appState->vkLastPostId; + long oldLastPostId = mgr->m_appState->vkLastLoadedPostId; if (posts.size() > 0) { spdlog::info("last vk post id is now {}", posts[0].id); - mgr->m_appState->vkLastPostId = posts[0].id; + //mgr->m_appState->vkLastLoadedPostId = posts[0].id; } std::vector aposts = mgr->to_abstract_posts(posts); + spdlog::info("looking for id {}, have {} - {}", oldLastPostId, aposts[0].id, aposts[aposts.size() - 1].id); if (mgr->drop_posts_older_than(aposts, oldLastPostId)) { spdlog::info("found last remembered VK post"); vkState.ready = true; @@ -143,14 +164,18 @@ void RepostManager::on_clients_ready() { vkState.posts.emplace_back(std::move(*i)); } vkState.needRequest = true; + if (vkState.ready) { + spdlog::debug("last loaded vk post id is now {}", vkState.posts[0].id); + mgr->m_appState->vkLastLoadedPostId = vkState.posts[0].id; + } fetch(); } void check_tg_posts(std::vector> posts) { spdlog::info("fetched {} TG posts", posts.size()); - long oldLastPostId = mgr->m_appState->tgLastPostId; + long oldLastPostId = mgr->m_appState->tgLastLoadedPostId; if (posts.size() > 0) { spdlog::info("last telegram post id is now {}", posts[0]->id_); - mgr->m_appState->tgLastPostId = posts[0]->id_; + //mgr->m_appState->tgLastLoadedPostId = posts[0]->id_; } std::vector aposts = mgr->to_abstract_posts(posts); if (mgr->drop_posts_older_than(aposts, oldLastPostId)) { @@ -162,6 +187,10 @@ void RepostManager::on_clients_ready() { tgState.posts.emplace_back(std::move(*i)); } tgState.needRequest = true; + if (tgState.ready) { + spdlog::debug("last loaded tg post id is now {}", tgState.posts[0].id); + mgr->m_appState->tgLastLoadedPostId = tgState.posts[0].id; + } fetch(); } }; @@ -183,7 +212,7 @@ void RepostManager::on_clients_ready() { } } spdlog::info("sorted {} posts", totalSize); - repost_all(mergedPosts); + enqueue_for_repost(mergedPosts); }; f->onError = [f](){ @@ -311,9 +340,50 @@ std::vector RepostManager::to_abstract_posts(std::vector posts) { +void RepostManager::enqueue_for_repost(std::vector posts) { for (auto &post : posts) { - auto content = td_api::make_object(td_api::make_object(post.text + "\nDate: " + std::to_string(post.date), std::vector>()), nullptr, false); - //m_tg.send_query(td_api::make_object(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), {}); + if (m_repostQueue.empty()) { + uv_timer_start(m_repostTimer, &RepostManager::repost_timer_callback, REPOST_INTERVAL, REPOST_INTERVAL); + } + m_repostQueue.push(post); } +} + +void RepostManager::repost_timer_callback(uv_timer_t *h) { + spdlog::debug("repost timer"); + auto self = reinterpret_cast(h->data); + if (!self->m_repostQueue.empty()) { + self->repost(self->m_repostQueue.front()); + self->m_repostQueue.pop(); + } + if (self->m_repostQueue.empty()) { + spdlog::debug("repost queue empty, stopping timer"); + uv_timer_stop(h); + } else { + spdlog::debug("{} posts left to repost", self->m_repostQueue.size()); + } +} + +void RepostManager::repost(AbstractPost &post) { + spdlog::debug("reposting (post length {})", post.text.length()); + std::string_view signature = posts::add_signature(post, m_appConfig); + spdlog::debug(post.text); + int signatureStart = post.text.length() - signature.length(); + int signatureLength = signature.length(); + spdlog::debug("post length {}, signature start {}, signature length {}", post.text.length(), signatureStart, signatureLength); + std::vector> entities; + //entities.push_back(std::move(td_api::make_object(signatureStart, signatureLength, td_api::make_object()))); + auto content = td_api::make_object(td_api::make_object(post.text, std::move(entities)), td_api::make_object(true, std::string(""), false, false, false), false); + m_tg.send_query(td_api::make_object(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), [this, postId = post.id, src = post.source](auto result){ + if (result->get_id() == td_api::error::ID) { + auto &err = (td_api::error&)*result; + spdlog::error("sendMessage error: {} {}", err.code_, err.message_); + uv_timer_stop(m_repostTimer); + } else { + if (src == posts::SRC_VK) + m_appState->vkLastPostId = postId; + else + m_appState->tgLastPostId = postId; + } + }); } \ No newline at end of file diff --git a/manager.h b/manager.h index d7f837e..4c18fe0 100644 --- a/manager.h +++ b/manager.h @@ -6,6 +6,7 @@ #include "tg.h" #include "vk.h" #include +#include #include namespace manager { @@ -17,9 +18,11 @@ namespace manager { public: RepostManager(uv_loop_t *eventLoop, tg::AuthCodeProvider tgCodeProvider, tg::PasswordProvider tgPasswordProvider, state::AppState *appState, config::AppConfig *config); RepostManager(RepostManager&) = delete; + ~RepostManager(); void start(); private: void on_clients_ready(); + void load_more_telegram_chats(); void collect_all_vk_posts(std::function)> callback); void collect_all_tg_posts(std::function>)> callback); @@ -36,11 +39,16 @@ namespace manager { std::vector to_abstract_posts(std::vector &posts); std::vector to_abstract_posts(std::vector> &posts); - void repost_all(std::vector posts); + void enqueue_for_repost(std::vector posts); + static void repost_timer_callback(uv_timer_t *h); + void repost(AbstractPost &post); state::AppState *m_appState; config::AppConfig *m_appConfig; vk::VKClient m_vk; tg::TelegramClient m_tg; + std::queue m_repostQueue; + uv_timer_t *m_repostTimer = nullptr; + uv_timer_t *m_checkTimer = nullptr; }; } diff --git a/posts.cpp b/posts.cpp index dd3e59e..9998461 100644 --- a/posts.cpp +++ b/posts.cpp @@ -1,3 +1,62 @@ #include "posts.h" +#include "config.h" +#include +#include +#include +#include -using namespace posts; \ No newline at end of file +using namespace posts; + +static std::regex vkRegex("«\\s*#[A-Za-z0-9\\-_.А-Яа-яЁё]+@mmcs_quotes\\s*»"); +static std::regex tgRegex1("#цитат(а|ы)"); +static std::regex tgRegex2("- "); + +bool posts::filter_and_transform(AbstractPost &post) { + if (post.source == SRC_VK) { + bool hasHashtag = std::regex_search(post.text, vkRegex); + if (!hasHashtag) return false; + return true; + } else { + if (std::regex_search(post.text, tgRegex1)) return true; + + int lastLinebreak = post.text.find_last_of('\n'); + std::string lastLine = post.text.substr(lastLinebreak == -1 ? 0 : lastLinebreak, post.text.size()); + if (std::regex_search(lastLine, tgRegex2)) { + post.text = post.text.substr(0, lastLinebreak); + return true; + } + return false; + } +} + +std::string_view posts::add_signature(AbstractPost &post, config::AppConfig *cfg) { + if (post.source == SRC_VK) { + return add_vk_signature(post.text, cfg); + } else { + return add_tg_signature(post.text, cfg); + } +} + +std::string_view posts::add_vk_signature(std::string &text, config::AppConfig *cfg) { + std::string shortname = std::holds_alternative(cfg->vkSource) ? std::get(cfg->vkSource) : cfg->vkSourceLink; + std::string signature = "\nРепостнуто автоматически из "; + if (!shortname.empty()) + signature += "vk.com/" + shortname; + else + signature += "VK"; + int oldPostLength = text.length() + 1; + text += signature; + return std::string_view(text.c_str() + oldPostLength, signature.length()); +} + +std::string_view posts::add_tg_signature(std::string &text, config::AppConfig *cfg) { + std::string shortname = cfg->tgSourceLink; + std::string signature = "\nРепостнуто автоматически из "; + if (!shortname.empty()) + signature += "t.me/" + shortname; + else + signature += "Telegram"; + int oldPostLength = text.length() + 1; + text += signature; + return std::string_view(text.c_str() + oldPostLength, signature.length()); +} \ No newline at end of file diff --git a/posts.h b/posts.h index 54669bd..2606c82 100644 --- a/posts.h +++ b/posts.h @@ -1,6 +1,8 @@ #pragma once +#include "config.h" #include +#include namespace posts { enum PostSource { @@ -14,4 +16,10 @@ namespace posts { std::string text; PostSource source; }; + + bool filter_and_transform(AbstractPost &post); + + std::string_view add_signature(AbstractPost &post, config::AppConfig *cfg); + std::string_view add_vk_signature(std::string &text, config::AppConfig *cfg); + std::string_view add_tg_signature(std::string &text, config::AppConfig *cfg); } \ No newline at end of file diff --git a/state.h b/state.h index a03b212..a2f1fb1 100644 --- a/state.h +++ b/state.h @@ -20,6 +20,9 @@ namespace state { int64_t tgLastPostId = 0; int64_t vkLastPostId = 0; + + int64_t tgLastLoadedPostId = 0; + int64_t vkLastLoadedPostId = 0; private: std::string m_saveFilename; }; diff --git a/vk.cpp b/vk.cpp index 7b8c222..6670b32 100644 --- a/vk.cpp +++ b/vk.cpp @@ -2,18 +2,22 @@ #include "curl/curl.h" #include "http.h" #include "spdlog/sinks/stdout_color_sinks.h" +#include #include #include #include using namespace vk; using namespace nlohmann; +using std::chrono::duration_cast, std::chrono::milliseconds, std::chrono::steady_clock; + const char *API_BASE_URL = "https://api.vk.com/method/"; const char *API_VERSION = "5.199"; const char *LOGGER_TAG = "vk"; +const long API_CALL_INTERVAL = 250; -VKClient::VKClient(uv_loop_t *eventLoop) : m_httpClient(eventLoop) { +VKClient::VKClient(uv_loop_t *eventLoop) : m_eventLoop(eventLoop), m_httpClient(eventLoop) { m_logger = spdlog::get(LOGGER_TAG); if (!m_logger) { m_logger = spdlog::stdout_color_mt(LOGGER_TAG); @@ -30,10 +34,39 @@ void VKClient::set_service_api_key(std::string key) { } void VKClient::get_posts(std::variant wall, int offset, int count, std::function, int)> callback) { + long delay = API_CALL_INTERVAL - duration_cast(steady_clock::now() - m_lastCallTime).count(); + if (delay <= 0) { + get_posts_now(wall, offset, count, callback); + } else { + m_logger->debug("delaying request by {} ms", delay); + struct params { + VKClient *self; + std::variant wall; + int offset, count; + std::function, int)> callback; + }; + uv_timer_t *timer = new uv_timer_t; + params *p = new params{ this, wall, offset, count, callback }; + timer->data = p; + + uv_timer_init(m_eventLoop, timer); + uv_timer_start(timer, [](uv_timer_t *h){ + uv_timer_stop(h); + uv_close((uv_handle_t*)h, [](uv_handle_t *h){ delete h; }); + + params *p = (params*)h->data; + p->self->get_posts(p->wall, p->offset, p->count, p->callback); + delete p; + }, delay, 0); + } +} + +void VKClient::get_posts_now(std::variant wall, int offset, int count, std::function, int)> callback) { if (!m_serviceApiKey) { m_logger->error("get_posts called without authorization"); return; } + m_lastCallTime = std::chrono::steady_clock::now(); std::string url(API_BASE_URL); url += "wall.get?access_token="; url += *m_serviceApiKey; diff --git a/vk.h b/vk.h index 3653269..f649209 100644 --- a/vk.h +++ b/vk.h @@ -2,8 +2,10 @@ #include "http.h" #include "spdlog/logger.h" #include "uv.h" +#include #include #include +#include #include #include @@ -32,9 +34,12 @@ namespace vk { void set_service_api_key(std::string key); void get_posts(std::variant wall, int offset, int count, std::function, int)> callback); + void get_posts_now(std::variant wall, int offset, int count, std::function, int)> callback); private: + uv_loop_t *m_eventLoop; HttpClient m_httpClient; std::optional m_serviceApiKey; std::shared_ptr m_logger; + std::chrono::steady_clock::time_point m_lastCallTime; }; } \ No newline at end of file