libcurl采用curl_multi_perform() + curl_multi_wait()方式实现异步高性能l发送数据的方法

前两篇文章

c/c++调用libcurl库发送http请求的两种基本用法

采用libuv的epoll方式实现的异步高性能libcurl发送数据的方法

讲述了采用libcurl发送数据的基础方法和高性能方法，基础方法较为容易但性能一般，高级方法的性能卓越但比较难理解，这里再给出一个保证性能的同时又相对较容易理解的方法，该方法最初是由facebook贡献，实现如下：

#define CURL_MULTI_NUM 5
#define CURL_MAX_WAIT_MSECS 30*1000

//将单条数据添加到总
static void add_multi_curl(CURLM* cm, Task* task) {
  struct curl_slist* headers = NULL;
  std::string tmp_str;

  //添加头信息
  tmp_str = "User-Agent: ";
  tmp_str += task->user_agent;
  headers = curl_slist_append(headers, tmp_str.c_str());
  tmp_str = "Accept-Language: ";
  tmp_str += task->language;
  headers = curl_slist_append(headers, tmp_str.c_str());
  tmp_str = "X-FORWORDED-FOR: ";
  tmp_str += task->x_forwarded_for;
  headers = curl_slist_append(headers, tmp_str.c_str());

  
  CURL* curl = curl_easy_init();
  //设置curl参数
  curl_easy_setopt(curl, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 60 * 72);
  curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
  curl_easy_setopt(curl, CURLOPT_TIMEOUT, 5);
  curl_easy_setopt(curl, CURLOPT_HEADER, 0L);
  curl_easy_setopt(curl, CURLOPT_URL, task->uri.c_str());
  curl_easy_setopt(curl, CURLOPT_PRIVATE, task->uri.c_str());
  curl_easy_setopt(curl, CURLOPT_VERBOSE, 0L);

  //添加单条curl信息到总信息中
  curl_multi_add_handle(cm, curl);
}

//发送总数据
bool send_multi_data(CURLM* cm) {
  CURL* eh = NULL;
  CURLMsg* msg = NULL;
  CURLcode return_code = CURLE_OK;
  int32_t still_running = 0;
  int32_t msgs_left = 0;
  int32_t http_status_code;
  const char* szUrl;

  //发送数据并用still_running监测数据是否发送完成，如果未发送完成则持续发送
  curl_multi_perform(cm, &still_running);
  do {
    int numfds = 0;
    int res = curl_multi_wait(cm, NULL, 0, CURL_MAX_WAIT_MSECS, &numfds);
    if (res != CURLM_OK) {
      LOG(WARNING) << "error: curl_multi_wait() returned" << res;
      return false;
    }
    curl_multi_perform(cm, &still_running);
  } while (still_running);

  //读取发送返回结果信息
  while ((msg = curl_multi_info_read(cm, &msgs_left))) {
	//是否读取成功
    if (msg->msg == CURLMSG_DONE) {
      eh = msg->easy_handle;

      return_code = msg->data.result;
	  //检测数据发送结果
      if (return_code != CURLE_OK) {
        LOG(WARNING) << "CURL error code: " << msg->data.result;
        continue;
      }

      //检测HTTP状态
      http_status_code = 0;
      szUrl = NULL;

      curl_easy_getinfo(eh, CURLINFO_RESPONSE_CODE, &http_status_code);
      curl_easy_getinfo(eh, CURLINFO_PRIVATE, &szUrl);

      if (http_status_code != 200) {
        LOG(WARNING) << "GET of" << szUrl << "returned http status code " << http_status_code;
      }

      curl_multi_remove_handle(cm, eh);
      curl_easy_cleanup(eh);
    } else {
      LOG(WARNING) << "error: after curl_multi_info_read(), CURLMsg= " << msg->msg;
    }
  }

  return true;
}

int main()
{
	
  int loop() {
    CURLM* cm = NULL;
    cm = curl_multi_init();
    int add_num = 0;
    while (!IsStoped()) {
      Task* task = GetTaskItem(); //获取待发送数据，采取消息队列的方式
      if (task == nullptr) {
		//如果获取不到数据，则将当前已获取的数据做一次性发送后等待
        if (add_num == 0) {
		  //循环等待消息队列
          WaitTask();
          continue;
        } else {
          send_multi_data(cm);
          add_num = 0;
        }
      } else {
		//数据量积累到CURL_MULTI_NUM后做一次性发送
        add_multi_curl(cm, task);
        add_num += 1;
        if (add_num >= CURL_MULTI_NUM) {
          send_multi_data(cm);
          add_num = 0;
        } else {
          continue;
        }
      }
    }
    curl_multi_cleanup(cm);
    return true;
  };
};