fix last post id tracking; add repost timer; add vk request timer; fix recent post scanning

config.cpp

@@ -1,5 +1,6 @@
 #include "config.h"
 #include "nlohmann/json.hpp"
+#include "spdlog/spdlog.h"
 #include <fstream>
 #include <string>
 
@@ -13,6 +14,8 @@ const char *JSON_KEY_TG_PHONE_NUMBER = "tg_phone_number";
 const char *JSON_KEY_VK_SOURCE = "vk_source";
 const char *JSON_KEY_TG_SOURCE_ID = "tg_source_id";
 const char *JSON_KEY_TG_DESTINATION_ID = "tg_destination_id";
+const char *JSON_KEY_VK_SOURCE_LINK = "vk_source_link";
+const char *JSON_KEY_TG_SOURCE_LINK = "tg_source_link";
 
 const char *ERR_INVALID_TYPE_VK_SERVICE_KEY = "vk_service_key must be a string";
 const char *ERR_INVALID_TYPE_TG_API_ID = "tg_api_id must be an integer or a string";
@@ -21,6 +24,8 @@ const char *ERR_INVALID_TYPE_TG_PHONE_NUMBER = "tg_phone_number must be a string
 const char *ERR_INVALID_TYPE_VK_SOURCE = "vk_source must be an integer or a string";
 const char *ERR_INVALID_TYPE_TG_SOURCE_ID = "tg_source_id must be an integer";
 const char *ERR_INVALID_TYPE_TG_DESTINATION_ID = "tg_destination_id must be an integer";
+const char *ERR_INVALID_TYPE_VK_SOURCE_LINK = "vk_source_link must be a string";
+const char *ERR_INVALID_TYPE_TG_SOURCE_LINK = "tg_source_link must be a string";
 
 AppConfig::AppConfig(const std::string &filename) {
   std::ifstream f(filename);
@@ -98,4 +103,22 @@ AppConfig::AppConfig(const std::string &filename) {
       throw InvalidConfigException(ERR_INVALID_TYPE_TG_DESTINATION_ID);
     }
   }
+
+  if (config.contains(JSON_KEY_VK_SOURCE_LINK)) {
+    json vkSourceLink_ = config[JSON_KEY_VK_SOURCE_LINK];
+    if (vkSourceLink_.type() == json::value_t::string) {
+      vkSourceLink = vkSourceLink_;
+    } else {
+      spdlog::warn("config: {}", ERR_INVALID_TYPE_VK_SOURCE_LINK);
+    }
+  }
+
+  if (config.contains(JSON_KEY_TG_SOURCE_LINK)) {
+    json tgSourceLink_ = config[JSON_KEY_TG_SOURCE_LINK];
+    if (tgSourceLink_.type() == json::value_t::string) {
+      tgSourceLink = tgSourceLink_;
+    } else {
+      spdlog::warn("config: {}", ERR_INVALID_TYPE_TG_SOURCE_LINK);
+    }
+  }
 }

config.h

@@ -18,6 +18,7 @@ namespace config {
     std::string tgApiHash;
     std::string tgPhoneNumber;
     std::variant<long, std::string> vkSource;
+    std::string vkSourceLink, tgSourceLink;
     long tgSourceId, tgDestinationId;
   };
 }

manager.cpp

@@ -1,7 +1,9 @@
 #include "manager.h"
+#include "posts.h"
 #include "spdlog/spdlog.h"
 #include "state.h"
 #include "td/telegram/td_api.h"
+#include "uv.h"
 #include "vk.h"
 #include <algorithm>
 #include <limits>
@@ -10,6 +12,8 @@
 
 using namespace manager;
 
+const unsigned long REPOST_INTERVAL = 2000;
+
 RepostManager::RepostManager(uv_loop_t *eventLoop, tg::AuthCodeProvider tgCodeProvider, tg::PasswordProvider tgPasswordProvider, state::AppState *appState, config::AppConfig *config)
  : m_vk(eventLoop), m_tg(eventLoop, config->tgApiId, config->tgApiHash, config->tgPhoneNumber) {
   m_appState = appState;
@@ -17,6 +21,17 @@ RepostManager::RepostManager(uv_loop_t *eventLoop, tg::AuthCodeProvider tgCodePr
   m_tg.authCodeProvider = tgCodeProvider;
   m_tg.passwordProvider = tgPasswordProvider;
   m_vk.set_service_api_key(config->vkServiceKey);
+
+  m_repostTimer = new uv_timer_t;
+  uv_timer_init(eventLoop, m_repostTimer);
+  m_repostTimer->data = this;
+}
+
+RepostManager::~RepostManager() {
+  if (m_repostTimer) {
+    uv_timer_stop(m_repostTimer);
+    uv_close((uv_handle_t*)m_repostTimer, [](uv_handle_t *h){ delete h; });
+  }
 }
 
 void RepostManager::load_more_telegram_chats() {
@@ -52,6 +67,9 @@ void RepostManager::start() {
 }
 
 void RepostManager::on_clients_ready() {
+  m_appState->vkLastLoadedPostId = m_appState->vkLastPostId;
+  m_appState->tgLastLoadedPostId = m_appState->tgLastPostId;
+
   struct new_post_fetcher {
     struct fetcher_state {
       bool ready = false;
@@ -71,12 +89,14 @@ void RepostManager::on_clients_ready() {
       }
 
       if (!vkState.ready && vkState.needRequest) {
-        if (mgr->m_appState->vkLastPostId != 0) {
+        if (mgr->m_appState->vkLastLoadedPostId != 0) {
           spdlog::info("fetching {} VK posts at offset {}", vkState.count, vkState.offset);
           vkState.needRequest = false;
-          mgr->collect_vk_posts_from(vkState.offset, vkState.count, [this](auto posts){ check_vk_posts(posts); });
+          mgr->collect_vk_posts_from(vkState.offset, vkState.count, [this](auto posts){
-          vkState.offset += vkState.count;
+            vkState.offset += posts.size();
-          vkState.count = vkState.count * 3 / 2;
+            vkState.count = vkState.count * 3 / 2;
+            check_vk_posts(posts);
+          });
         } else {
           spdlog::info("fetching all VK posts");
           vkState.needRequest = false;
@@ -84,7 +104,7 @@ void RepostManager::on_clients_ready() {
             spdlog::info("fetched all {} VK posts", posts.size());
             if (posts.size() > 0) {
               spdlog::info("last vk post id is now {}", posts[0].id);
-              mgr->m_appState->vkLastPostId = posts[0].id;
+              mgr->m_appState->vkLastLoadedPostId = posts[0].id;
             }
             vkState.ready = true;
             std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts);
@@ -98,7 +118,7 @@ void RepostManager::on_clients_ready() {
       }
 
       if (!tgState.ready && tgState.needRequest) {
-        if (mgr->m_appState->tgLastPostId != 0) {
+        if (mgr->m_appState->tgLastLoadedPostId != 0) {
           spdlog::info("fetching {} TG posts starting from #{}", tgState.count, tgState.offset);
           tgState.needRequest = false;
           mgr->collect_tg_posts_from(tgState.offset, tgState.count, [this](auto posts){
@@ -113,7 +133,7 @@ void RepostManager::on_clients_ready() {
             spdlog::info("fetched all {} TG posts", posts.size());
             if (posts.size() > 0) {
               spdlog::info("last telegram post id is now {}", posts[0]->id_);
-              mgr->m_appState->tgLastPostId = posts[0]->id_;
+              mgr->m_appState->tgLastLoadedPostId = posts[0]->id_;
             }
             tgState.ready = true;
             std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts);
@@ -128,12 +148,13 @@ void RepostManager::on_clients_ready() {
     }
     void check_vk_posts(std::vector<vk::Post> posts) {
       spdlog::info("fetched {} VK posts", posts.size());
-      long oldLastPostId = mgr->m_appState->vkLastPostId;
+      long oldLastPostId = mgr->m_appState->vkLastLoadedPostId;
       if (posts.size() > 0) {
         spdlog::info("last vk post id is now {}", posts[0].id);
-        mgr->m_appState->vkLastPostId = posts[0].id;
+        //mgr->m_appState->vkLastLoadedPostId = posts[0].id;
       }
       std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts);
+      spdlog::info("looking for id {}, have {} - {}", oldLastPostId, aposts[0].id, aposts[aposts.size() - 1].id);
       if (mgr->drop_posts_older_than(aposts, oldLastPostId)) {
         spdlog::info("found last remembered VK post");
         vkState.ready = true;
@@ -143,14 +164,18 @@ void RepostManager::on_clients_ready() {
         vkState.posts.emplace_back(std::move(*i));
       }
       vkState.needRequest = true;
+      if (vkState.ready) {
+        spdlog::debug("last loaded vk post id is now {}", vkState.posts[0].id);
+        mgr->m_appState->vkLastLoadedPostId = vkState.posts[0].id;
+      }
       fetch();
     }
     void check_tg_posts(std::vector<td::tl::unique_ptr<td_api::message>> posts) {
       spdlog::info("fetched {} TG posts", posts.size());
-      long oldLastPostId = mgr->m_appState->tgLastPostId;
+      long oldLastPostId = mgr->m_appState->tgLastLoadedPostId;
       if (posts.size() > 0) {
         spdlog::info("last telegram post id is now {}", posts[0]->id_);
-        mgr->m_appState->tgLastPostId = posts[0]->id_;
+        //mgr->m_appState->tgLastLoadedPostId = posts[0]->id_;
       }
       std::vector<AbstractPost> aposts = mgr->to_abstract_posts(posts);
       if (mgr->drop_posts_older_than(aposts, oldLastPostId)) {
@@ -162,6 +187,10 @@ void RepostManager::on_clients_ready() {
         tgState.posts.emplace_back(std::move(*i));
       }
       tgState.needRequest = true;
+      if (tgState.ready) {
+        spdlog::debug("last loaded tg post id is now {}", tgState.posts[0].id);
+        mgr->m_appState->tgLastLoadedPostId = tgState.posts[0].id;
+      }
       fetch();
     }
   };
@@ -183,7 +212,7 @@ void RepostManager::on_clients_ready() {
       }
     }
     spdlog::info("sorted {} posts", totalSize);
-    repost_all(mergedPosts);
+    enqueue_for_repost(mergedPosts);
   };
   
   f->onError = [f](){
@@ -311,9 +340,50 @@ std::vector<AbstractPost> RepostManager::to_abstract_posts(std::vector<td::tl::u
 }
 
 // TODO timer
-void RepostManager::repost_all(std::vector<AbstractPost> posts) {
+void RepostManager::enqueue_for_repost(std::vector<AbstractPost> posts) {
   for (auto &post : posts) {
-    auto content = td_api::make_object<td_api::inputMessageText>(td_api::make_object<td_api::formattedText>(post.text + "\nDate: " + std::to_string(post.date), std::vector<td_api::object_ptr<td_api::textEntity>>()), nullptr, false);
+    if (m_repostQueue.empty()) {
-    //m_tg.send_query(td_api::make_object<td_api::sendMessage>(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), {});
+      uv_timer_start(m_repostTimer, &RepostManager::repost_timer_callback, REPOST_INTERVAL, REPOST_INTERVAL);
+    }
+    m_repostQueue.push(post);
   }
 }
+
+void RepostManager::repost_timer_callback(uv_timer_t *h) {
+  spdlog::debug("repost timer");
+  auto self = reinterpret_cast<RepostManager*>(h->data);
+  if (!self->m_repostQueue.empty()) {
+    self->repost(self->m_repostQueue.front());
+    self->m_repostQueue.pop();
+  }
+  if (self->m_repostQueue.empty()) {
+    spdlog::debug("repost queue empty, stopping timer");
+    uv_timer_stop(h);
+  } else {
+    spdlog::debug("{} posts left to repost", self->m_repostQueue.size());
+  }
+}
+
+void RepostManager::repost(AbstractPost &post) {
+  spdlog::debug("reposting (post length {})", post.text.length());
+  std::string_view signature = posts::add_signature(post, m_appConfig);
+  spdlog::debug(post.text);
+  int signatureStart = post.text.length() - signature.length();
+  int signatureLength = signature.length();
+  spdlog::debug("post length {}, signature start {}, signature length {}", post.text.length(), signatureStart, signatureLength);
+  std::vector<td::tl::unique_ptr<td_api::textEntity>> entities;
+  //entities.push_back(std::move(td_api::make_object<td_api::textEntity>(signatureStart, signatureLength, td_api::make_object<td_api::textEntityTypeSpoiler>())));
+  auto content = td_api::make_object<td_api::inputMessageText>(td_api::make_object<td_api::formattedText>(post.text, std::move(entities)), td_api::make_object<td_api::linkPreviewOptions>(true, std::string(""), false, false, false), false);
+  m_tg.send_query(td_api::make_object<td_api::sendMessage>(m_appConfig->tgDestinationId, 0, nullptr, nullptr, nullptr, std::move(content)), [this, postId = post.id, src = post.source](auto result){
+    if (result->get_id() == td_api::error::ID) {
+      auto &err = (td_api::error&)*result;
+      spdlog::error("sendMessage error: {} {}", err.code_, err.message_);
+      uv_timer_stop(m_repostTimer);
+    } else {
+      if (src == posts::SRC_VK)
+        m_appState->vkLastPostId = postId;
+      else
+        m_appState->tgLastPostId = postId;
+    }
+  });
+}

manager.h

@@ -6,6 +6,7 @@
 #include "tg.h"
 #include "vk.h"
 #include <functional>
+#include <queue>
 #include <vector>
 
 namespace manager {
@@ -17,9 +18,11 @@ namespace manager {
   public:
     RepostManager(uv_loop_t *eventLoop, tg::AuthCodeProvider tgCodeProvider, tg::PasswordProvider tgPasswordProvider, state::AppState *appState, config::AppConfig *config);
     RepostManager(RepostManager&) = delete;
+    ~RepostManager();
     void start();
   private:
     void on_clients_ready();
+    void load_more_telegram_chats();
 
     void collect_all_vk_posts(std::function<void(std::vector<vk::Post>)> callback);
     void collect_all_tg_posts(std::function<void(std::vector<td::tl::unique_ptr<td_api::message>>)> callback);
@@ -36,11 +39,16 @@ namespace manager {
     std::vector<AbstractPost> to_abstract_posts(std::vector<vk::Post> &posts);
     std::vector<AbstractPost> to_abstract_posts(std::vector<td::tl::unique_ptr<td_api::message>> &posts);
 
-    void repost_all(std::vector<AbstractPost> posts);
+    void enqueue_for_repost(std::vector<AbstractPost> posts);
+    static void repost_timer_callback(uv_timer_t *h);
+    void repost(AbstractPost &post);
 
     state::AppState *m_appState;
     config::AppConfig *m_appConfig;
     vk::VKClient m_vk;
     tg::TelegramClient m_tg;
+    std::queue<AbstractPost> m_repostQueue;
+    uv_timer_t *m_repostTimer = nullptr;
+    uv_timer_t *m_checkTimer = nullptr;
   };
 }

posts.cpp

@@ -1,3 +1,62 @@
 #include "posts.h"
+#include "config.h"
+#include <regex>
+#include <string>
+#include <string_view>
+#include <variant>
 
 using namespace posts;
+
+static std::regex vkRegex("«\\s*#[A-Za-z0-9\\-_.А-Яа-яЁё]+@mmcs_quotes\\s*»");
+static std::regex tgRegex1("#цитат(а|ы)");
+static std::regex tgRegex2("- ");
+
+bool posts::filter_and_transform(AbstractPost &post) {
+  if (post.source == SRC_VK) {
+    bool hasHashtag = std::regex_search(post.text, vkRegex);
+    if (!hasHashtag) return false;
+    return true;
+  } else {
+    if (std::regex_search(post.text, tgRegex1)) return true;
+
+    int lastLinebreak = post.text.find_last_of('\n');
+    std::string lastLine = post.text.substr(lastLinebreak == -1 ? 0 : lastLinebreak, post.text.size());
+    if (std::regex_search(lastLine, tgRegex2)) {
+      post.text = post.text.substr(0, lastLinebreak);
+      return true;
+    }
+    return false;
+  }
+}
+
+std::string_view posts::add_signature(AbstractPost &post, config::AppConfig *cfg) {
+  if (post.source == SRC_VK) {
+    return add_vk_signature(post.text, cfg);
+  } else {
+    return add_tg_signature(post.text, cfg);
+  }
+}
+
+std::string_view posts::add_vk_signature(std::string &text, config::AppConfig *cfg) {
+  std::string shortname = std::holds_alternative<std::string>(cfg->vkSource) ? std::get<std::string>(cfg->vkSource) : cfg->vkSourceLink;
+  std::string signature = "\nРепостнуто автоматически из ";
+  if (!shortname.empty())
+    signature += "vk.com/" + shortname;
+  else
+    signature += "VK";
+  int oldPostLength = text.length() + 1;
+  text += signature;
+  return std::string_view(text.c_str() + oldPostLength, signature.length());
+}
+
+std::string_view posts::add_tg_signature(std::string &text, config::AppConfig *cfg) {
+  std::string shortname = cfg->tgSourceLink;
+  std::string signature = "\nРепостнуто автоматически из ";
+  if (!shortname.empty())
+    signature += "t.me/" + shortname;
+  else
+    signature += "Telegram";
+  int oldPostLength = text.length() + 1;
+  text += signature;
+  return std::string_view(text.c_str() + oldPostLength, signature.length());
+}

posts.h

@@ -1,6 +1,8 @@
 #pragma once
 
+#include "config.h"
 #include <string>
+#include <string_view>
 
 namespace posts {
   enum PostSource {
@@ -14,4 +16,10 @@ namespace posts {
     std::string text;
     PostSource source;
   };
+
+  bool filter_and_transform(AbstractPost &post);
+
+  std::string_view add_signature(AbstractPost &post, config::AppConfig *cfg);
+  std::string_view add_vk_signature(std::string &text, config::AppConfig *cfg);
+  std::string_view add_tg_signature(std::string &text, config::AppConfig *cfg);
 }

state.h

@@ -20,6 +20,9 @@ namespace state {
 
     int64_t tgLastPostId = 0;
     int64_t vkLastPostId = 0;
+
+    int64_t tgLastLoadedPostId = 0;
+    int64_t vkLastLoadedPostId = 0;
   private:
     std::string m_saveFilename;
   };

vk.cpp

@@ -2,18 +2,22 @@
 #include "curl/curl.h"
 #include "http.h"
 #include "spdlog/sinks/stdout_color_sinks.h"
+#include <chrono>
 #include <limits>
 #include <memory>
 #include <nlohmann/json.hpp>
 
 using namespace vk;
 using namespace nlohmann;
+using std::chrono::duration_cast, std::chrono::milliseconds, std::chrono::steady_clock;
+
 
 const char *API_BASE_URL = "https://api.vk.com/method/";
 const char *API_VERSION = "5.199";
 const char *LOGGER_TAG = "vk";
+const long API_CALL_INTERVAL = 250;
 
-VKClient::VKClient(uv_loop_t *eventLoop) : m_httpClient(eventLoop) {
+VKClient::VKClient(uv_loop_t *eventLoop) : m_eventLoop(eventLoop), m_httpClient(eventLoop) {
   m_logger = spdlog::get(LOGGER_TAG);
   if (!m_logger) {
     m_logger = spdlog::stdout_color_mt(LOGGER_TAG);
@@ -30,10 +34,39 @@ void VKClient::set_service_api_key(std::string key) {
 }
 
 void VKClient::get_posts(std::variant<long, std::string> wall, int offset, int count, std::function<void(std::optional<WallChunk>, int)> callback) {
+  long delay = API_CALL_INTERVAL - duration_cast<milliseconds>(steady_clock::now() - m_lastCallTime).count();
+  if (delay <= 0) {
+    get_posts_now(wall, offset, count, callback);
+  } else {
+    m_logger->debug("delaying request by {} ms", delay);
+    struct params {
+      VKClient *self;
+      std::variant<long, std::string> wall;
+      int offset, count;
+      std::function<void(std::optional<WallChunk>, int)> callback;
+    };
+    uv_timer_t *timer = new uv_timer_t;
+    params *p = new params{ this, wall, offset, count, callback };
+    timer->data = p;
+    
+    uv_timer_init(m_eventLoop, timer);
+    uv_timer_start(timer, [](uv_timer_t *h){
+      uv_timer_stop(h);
+      uv_close((uv_handle_t*)h, [](uv_handle_t *h){ delete h; });
+      
+      params *p = (params*)h->data;
+      p->self->get_posts(p->wall, p->offset, p->count, p->callback);
+      delete p;
+    }, delay, 0);
+  }
+}
+
+void VKClient::get_posts_now(std::variant<long, std::string> wall, int offset, int count, std::function<void(std::optional<WallChunk>, int)> callback) {
   if (!m_serviceApiKey) {
     m_logger->error("get_posts called without authorization");
     return;
   }
+  m_lastCallTime = std::chrono::steady_clock::now();
   std::string url(API_BASE_URL);
   url += "wall.get?access_token=";
   url += *m_serviceApiKey;

vk.h

@@ -2,8 +2,10 @@
 #include "http.h"
 #include "spdlog/logger.h"
 #include "uv.h"
+#include <chrono>
 #include <memory>
 #include <optional>
+#include <ratio>
 #include <string>
 #include <variant>
 
@@ -32,9 +34,12 @@ namespace vk {
     
     void set_service_api_key(std::string key);
     void get_posts(std::variant<long, std::string> wall, int offset, int count, std::function<void(std::optional<WallChunk>, int)> callback);
+    void get_posts_now(std::variant<long, std::string> wall, int offset, int count, std::function<void(std::optional<WallChunk>, int)> callback);
   private:
+    uv_loop_t *m_eventLoop;
     HttpClient m_httpClient;
     std::optional<std::string> m_serviceApiKey;
     std::shared_ptr<spdlog::logger> m_logger;
+    std::chrono::steady_clock::time_point m_lastCallTime;
   };
 }