1bd030d0aSAppaRao Puli /* 2bd030d0aSAppaRao Puli // Copyright (c) 2020 Intel Corporation 3bd030d0aSAppaRao Puli // 4bd030d0aSAppaRao Puli // Licensed under the Apache License, Version 2.0 (the "License"); 5bd030d0aSAppaRao Puli // you may not use this file except in compliance with the License. 6bd030d0aSAppaRao Puli // You may obtain a copy of the License at 7bd030d0aSAppaRao Puli // 8bd030d0aSAppaRao Puli // http://www.apache.org/licenses/LICENSE-2.0 9bd030d0aSAppaRao Puli // 10bd030d0aSAppaRao Puli // Unless required by applicable law or agreed to in writing, software 11bd030d0aSAppaRao Puli // distributed under the License is distributed on an "AS IS" BASIS, 12bd030d0aSAppaRao Puli // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13bd030d0aSAppaRao Puli // See the License for the specific language governing permissions and 14bd030d0aSAppaRao Puli // limitations under the License. 15bd030d0aSAppaRao Puli */ 16bd030d0aSAppaRao Puli #pragma once 17bb49eb5cSEd Tanous #include <boost/asio/io_context.hpp> 1829a82b08SSunitha Harish #include <boost/asio/ip/address.hpp> 1929a82b08SSunitha Harish #include <boost/asio/ip/basic_endpoint.hpp> 20bb49eb5cSEd Tanous #include <boost/asio/ip/tcp.hpp> 21d43cd0caSEd Tanous #include <boost/asio/steady_timer.hpp> 22d43cd0caSEd Tanous #include <boost/beast/core/flat_buffer.hpp> 23bb49eb5cSEd Tanous #include <boost/beast/core/flat_static_buffer.hpp> 24d43cd0caSEd Tanous #include <boost/beast/core/tcp_stream.hpp> 25d43cd0caSEd Tanous #include <boost/beast/http/message.hpp> 26bb49eb5cSEd Tanous #include <boost/beast/http/parser.hpp> 27bb49eb5cSEd Tanous #include <boost/beast/http/read.hpp> 28bb49eb5cSEd Tanous #include <boost/beast/http/string_body.hpp> 29bb49eb5cSEd Tanous #include <boost/beast/http/write.hpp> 30bd030d0aSAppaRao Puli #include <boost/beast/version.hpp> 31f52c03c1SCarson Labrado #include <boost/container/devector.hpp> 32bb49eb5cSEd Tanous #include <boost/system/error_code.hpp> 33bb49eb5cSEd Tanous #include <http/http_response.hpp> 3429a82b08SSunitha Harish #include <include/async_resolve.hpp> 35bb49eb5cSEd Tanous #include <logging.hpp> 361214b7e7SGunnar Mills 37bd030d0aSAppaRao Puli #include <cstdlib> 38bd030d0aSAppaRao Puli #include <functional> 39bd030d0aSAppaRao Puli #include <iostream> 40bd030d0aSAppaRao Puli #include <memory> 412a5689a7SAppaRao Puli #include <queue> 42bd030d0aSAppaRao Puli #include <string> 43bd030d0aSAppaRao Puli 44bd030d0aSAppaRao Puli namespace crow 45bd030d0aSAppaRao Puli { 46bd030d0aSAppaRao Puli 47f52c03c1SCarson Labrado // It is assumed that the BMC should be able to handle 4 parallel connections 48f52c03c1SCarson Labrado constexpr uint8_t maxPoolSize = 4; 49f52c03c1SCarson Labrado constexpr uint8_t maxRequestQueueSize = 50; 504d94272fSCarson Labrado constexpr unsigned int httpReadBodyLimit = 16384; 514d94272fSCarson Labrado constexpr unsigned int httpReadBufferSize = 4096; 522a5689a7SAppaRao Puli 53bd030d0aSAppaRao Puli enum class ConnState 54bd030d0aSAppaRao Puli { 552a5689a7SAppaRao Puli initialized, 5629a82b08SSunitha Harish resolveInProgress, 5729a82b08SSunitha Harish resolveFailed, 582a5689a7SAppaRao Puli connectInProgress, 592a5689a7SAppaRao Puli connectFailed, 60bd030d0aSAppaRao Puli connected, 612a5689a7SAppaRao Puli sendInProgress, 622a5689a7SAppaRao Puli sendFailed, 636eaa1d2fSSunitha Harish recvInProgress, 642a5689a7SAppaRao Puli recvFailed, 652a5689a7SAppaRao Puli idle, 666eaa1d2fSSunitha Harish closeInProgress, 67fe44eb0bSAyushi Smriti closed, 686eaa1d2fSSunitha Harish suspended, 696eaa1d2fSSunitha Harish terminated, 706eaa1d2fSSunitha Harish abortConnection, 716eaa1d2fSSunitha Harish retry 72bd030d0aSAppaRao Puli }; 73bd030d0aSAppaRao Puli 74a7a80296SCarson Labrado static inline boost::system::error_code 75a7a80296SCarson Labrado defaultRetryHandler(unsigned int respCode) 76a7a80296SCarson Labrado { 77a7a80296SCarson Labrado // As a default, assume 200X is alright 78a7a80296SCarson Labrado BMCWEB_LOG_DEBUG << "Using default check for response code validity"; 79a7a80296SCarson Labrado if ((respCode < 200) || (respCode >= 300)) 80a7a80296SCarson Labrado { 81a7a80296SCarson Labrado return boost::system::errc::make_error_code( 82a7a80296SCarson Labrado boost::system::errc::result_out_of_range); 83a7a80296SCarson Labrado } 84a7a80296SCarson Labrado 85a7a80296SCarson Labrado // Return 0 if the response code is valid 86a7a80296SCarson Labrado return boost::system::errc::make_error_code(boost::system::errc::success); 87a7a80296SCarson Labrado }; 88a7a80296SCarson Labrado 89f52c03c1SCarson Labrado // We need to allow retry information to be set before a message has been sent 90f52c03c1SCarson Labrado // and a connection pool has been created 91f52c03c1SCarson Labrado struct RetryPolicyData 92f52c03c1SCarson Labrado { 93f52c03c1SCarson Labrado uint32_t maxRetryAttempts = 5; 94f52c03c1SCarson Labrado std::chrono::seconds retryIntervalSecs = std::chrono::seconds(0); 95f52c03c1SCarson Labrado std::string retryPolicyAction = "TerminateAfterRetries"; 96a7a80296SCarson Labrado std::function<boost::system::error_code(unsigned int respCode)> 97a7a80296SCarson Labrado invalidResp = defaultRetryHandler; 98f52c03c1SCarson Labrado }; 99f52c03c1SCarson Labrado 100f52c03c1SCarson Labrado struct PendingRequest 101f52c03c1SCarson Labrado { 102244256ccSCarson Labrado boost::beast::http::request<boost::beast::http::string_body> req; 103039a47e3SCarson Labrado std::function<void(bool, uint32_t, Response&)> callback; 104f52c03c1SCarson Labrado RetryPolicyData retryPolicy; 105039a47e3SCarson Labrado PendingRequest( 1068a592810SEd Tanous boost::beast::http::request<boost::beast::http::string_body>&& reqIn, 1078a592810SEd Tanous const std::function<void(bool, uint32_t, Response&)>& callbackIn, 1088a592810SEd Tanous const RetryPolicyData& retryPolicyIn) : 1098a592810SEd Tanous req(std::move(reqIn)), 1108a592810SEd Tanous callback(callbackIn), retryPolicy(retryPolicyIn) 111f52c03c1SCarson Labrado {} 112f52c03c1SCarson Labrado }; 113f52c03c1SCarson Labrado 114f52c03c1SCarson Labrado class ConnectionInfo : public std::enable_shared_from_this<ConnectionInfo> 115bd030d0aSAppaRao Puli { 116bd030d0aSAppaRao Puli private: 117f52c03c1SCarson Labrado ConnState state = ConnState::initialized; 118f52c03c1SCarson Labrado uint32_t retryCount = 0; 119f52c03c1SCarson Labrado bool runningTimer = false; 120f52c03c1SCarson Labrado std::string subId; 121f52c03c1SCarson Labrado std::string host; 122f52c03c1SCarson Labrado uint16_t port; 123f52c03c1SCarson Labrado uint32_t connId; 124f52c03c1SCarson Labrado 125f52c03c1SCarson Labrado // Retry policy information 126f52c03c1SCarson Labrado // This should be updated before each message is sent 127f52c03c1SCarson Labrado RetryPolicyData retryPolicy; 128f52c03c1SCarson Labrado 129f52c03c1SCarson Labrado // Data buffers 130bd030d0aSAppaRao Puli boost::beast::http::request<boost::beast::http::string_body> req; 1316eaa1d2fSSunitha Harish std::optional< 1326eaa1d2fSSunitha Harish boost::beast::http::response_parser<boost::beast::http::string_body>> 1336eaa1d2fSSunitha Harish parser; 1344d94272fSCarson Labrado boost::beast::flat_static_buffer<httpReadBufferSize> buffer; 135039a47e3SCarson Labrado Response res; 1366eaa1d2fSSunitha Harish 137f52c03c1SCarson Labrado // Ascync callables 138039a47e3SCarson Labrado std::function<void(bool, uint32_t, Response&)> callback; 139f52c03c1SCarson Labrado crow::async_resolve::Resolver resolver; 140f52c03c1SCarson Labrado boost::beast::tcp_stream conn; 141f52c03c1SCarson Labrado boost::asio::steady_timer timer; 14284b35604SEd Tanous 143f52c03c1SCarson Labrado friend class ConnectionPool; 144bd030d0aSAppaRao Puli 14529a82b08SSunitha Harish void doResolve() 14629a82b08SSunitha Harish { 14729a82b08SSunitha Harish state = ConnState::resolveInProgress; 148f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Trying to resolve: " << host << ":" 149f52c03c1SCarson Labrado << std::to_string(port) 150f52c03c1SCarson Labrado << ", id: " << std::to_string(connId); 15129a82b08SSunitha Harish 15229a82b08SSunitha Harish auto respHandler = 15329a82b08SSunitha Harish [self(shared_from_this())]( 15429a82b08SSunitha Harish const boost::beast::error_code ec, 15529a82b08SSunitha Harish const std::vector<boost::asio::ip::tcp::endpoint>& 15629a82b08SSunitha Harish endpointList) { 15726f6976fSEd Tanous if (ec || (endpointList.empty())) 15829a82b08SSunitha Harish { 15929a82b08SSunitha Harish BMCWEB_LOG_ERROR << "Resolve failed: " << ec.message(); 16029a82b08SSunitha Harish self->state = ConnState::resolveFailed; 161f52c03c1SCarson Labrado self->waitAndRetry(); 16229a82b08SSunitha Harish return; 16329a82b08SSunitha Harish } 164f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Resolved " << self->host << ":" 165f52c03c1SCarson Labrado << std::to_string(self->port) 166f52c03c1SCarson Labrado << ", id: " << std::to_string(self->connId); 16729a82b08SSunitha Harish self->doConnect(endpointList); 16829a82b08SSunitha Harish }; 169f52c03c1SCarson Labrado 17029a82b08SSunitha Harish resolver.asyncResolve(host, port, std::move(respHandler)); 17129a82b08SSunitha Harish } 17229a82b08SSunitha Harish 17329a82b08SSunitha Harish void doConnect( 17429a82b08SSunitha Harish const std::vector<boost::asio::ip::tcp::endpoint>& endpointList) 175bd030d0aSAppaRao Puli { 1762a5689a7SAppaRao Puli state = ConnState::connectInProgress; 1772a5689a7SAppaRao Puli 178f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Trying to connect to: " << host << ":" 179f52c03c1SCarson Labrado << std::to_string(port) 180f52c03c1SCarson Labrado << ", id: " << std::to_string(connId); 181b00dcc27SEd Tanous 18229a82b08SSunitha Harish conn.expires_after(std::chrono::seconds(30)); 183002d39b4SEd Tanous conn.async_connect(endpointList, 184002d39b4SEd Tanous [self(shared_from_this())]( 18529a82b08SSunitha Harish const boost::beast::error_code ec, 18629a82b08SSunitha Harish const boost::asio::ip::tcp::endpoint& endpoint) { 1872a5689a7SAppaRao Puli if (ec) 1882a5689a7SAppaRao Puli { 189002d39b4SEd Tanous BMCWEB_LOG_ERROR << "Connect " << endpoint.address().to_string() 190002d39b4SEd Tanous << ":" << std::to_string(endpoint.port()) 191f52c03c1SCarson Labrado << ", id: " << std::to_string(self->connId) 1922a5689a7SAppaRao Puli << " failed: " << ec.message(); 1932a5689a7SAppaRao Puli self->state = ConnState::connectFailed; 194f52c03c1SCarson Labrado self->waitAndRetry(); 1952a5689a7SAppaRao Puli return; 1962a5689a7SAppaRao Puli } 197f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG 198f52c03c1SCarson Labrado << "Connected to: " << endpoint.address().to_string() << ":" 199f52c03c1SCarson Labrado << std::to_string(endpoint.port()) 200f52c03c1SCarson Labrado << ", id: " << std::to_string(self->connId); 2016eaa1d2fSSunitha Harish self->state = ConnState::connected; 202f52c03c1SCarson Labrado self->sendMessage(); 2032a5689a7SAppaRao Puli }); 2042a5689a7SAppaRao Puli } 2052a5689a7SAppaRao Puli 206f52c03c1SCarson Labrado void sendMessage() 2072a5689a7SAppaRao Puli { 2082a5689a7SAppaRao Puli state = ConnState::sendInProgress; 2092a5689a7SAppaRao Puli 210bd030d0aSAppaRao Puli // Set a timeout on the operation 211bd030d0aSAppaRao Puli conn.expires_after(std::chrono::seconds(30)); 212bd030d0aSAppaRao Puli 213bd030d0aSAppaRao Puli // Send the HTTP request to the remote host 214bd030d0aSAppaRao Puli boost::beast::http::async_write( 215bd030d0aSAppaRao Puli conn, req, 2162a5689a7SAppaRao Puli [self(shared_from_this())](const boost::beast::error_code& ec, 217bd030d0aSAppaRao Puli const std::size_t& bytesTransferred) { 218bd030d0aSAppaRao Puli if (ec) 219bd030d0aSAppaRao Puli { 220002d39b4SEd Tanous BMCWEB_LOG_ERROR << "sendMessage() failed: " << ec.message(); 2212a5689a7SAppaRao Puli self->state = ConnState::sendFailed; 222f52c03c1SCarson Labrado self->waitAndRetry(); 223bd030d0aSAppaRao Puli return; 224bd030d0aSAppaRao Puli } 225bd030d0aSAppaRao Puli BMCWEB_LOG_DEBUG << "sendMessage() bytes transferred: " 226bd030d0aSAppaRao Puli << bytesTransferred; 227bd030d0aSAppaRao Puli boost::ignore_unused(bytesTransferred); 228bd030d0aSAppaRao Puli 2292a5689a7SAppaRao Puli self->recvMessage(); 230bd030d0aSAppaRao Puli }); 231bd030d0aSAppaRao Puli } 232bd030d0aSAppaRao Puli 233bd030d0aSAppaRao Puli void recvMessage() 234bd030d0aSAppaRao Puli { 2356eaa1d2fSSunitha Harish state = ConnState::recvInProgress; 2366eaa1d2fSSunitha Harish 2376eaa1d2fSSunitha Harish parser.emplace(std::piecewise_construct, std::make_tuple()); 2386eaa1d2fSSunitha Harish parser->body_limit(httpReadBodyLimit); 2396eaa1d2fSSunitha Harish 240bd030d0aSAppaRao Puli // Receive the HTTP response 241bd030d0aSAppaRao Puli boost::beast::http::async_read( 2426eaa1d2fSSunitha Harish conn, buffer, *parser, 2432a5689a7SAppaRao Puli [self(shared_from_this())](const boost::beast::error_code& ec, 244bd030d0aSAppaRao Puli const std::size_t& bytesTransferred) { 245bd030d0aSAppaRao Puli if (ec) 246bd030d0aSAppaRao Puli { 247002d39b4SEd Tanous BMCWEB_LOG_ERROR << "recvMessage() failed: " << ec.message(); 2482a5689a7SAppaRao Puli self->state = ConnState::recvFailed; 249f52c03c1SCarson Labrado self->waitAndRetry(); 250bd030d0aSAppaRao Puli return; 251bd030d0aSAppaRao Puli } 252bd030d0aSAppaRao Puli BMCWEB_LOG_DEBUG << "recvMessage() bytes transferred: " 253bd030d0aSAppaRao Puli << bytesTransferred; 2546eaa1d2fSSunitha Harish BMCWEB_LOG_DEBUG << "recvMessage() data: " 2558cc8edecSEd Tanous << self->parser->get().body(); 256bd030d0aSAppaRao Puli 2576eaa1d2fSSunitha Harish unsigned int respCode = self->parser->get().result_int(); 2586eaa1d2fSSunitha Harish BMCWEB_LOG_DEBUG << "recvMessage() Header Response Code: " 2596eaa1d2fSSunitha Harish << respCode; 2606eaa1d2fSSunitha Harish 261a7a80296SCarson Labrado // Make sure the received response code is valid as defined by 262a7a80296SCarson Labrado // the associated retry policy 263a7a80296SCarson Labrado if (self->retryPolicy.invalidResp(respCode)) 2646eaa1d2fSSunitha Harish { 2656eaa1d2fSSunitha Harish // The listener failed to receive the Sent-Event 266002d39b4SEd Tanous BMCWEB_LOG_ERROR << "recvMessage() Listener Failed to " 2677adb85acSSunitha Harish "receive Sent-Event. Header Response Code: " 2687adb85acSSunitha Harish << respCode; 2696eaa1d2fSSunitha Harish self->state = ConnState::recvFailed; 270f52c03c1SCarson Labrado self->waitAndRetry(); 2716eaa1d2fSSunitha Harish return; 2726eaa1d2fSSunitha Harish } 273bd030d0aSAppaRao Puli 274f52c03c1SCarson Labrado // Send is successful 275f52c03c1SCarson Labrado // Reset the counter just in case this was after retrying 276f52c03c1SCarson Labrado self->retryCount = 0; 2776eaa1d2fSSunitha Harish 2786eaa1d2fSSunitha Harish // Keep the connection alive if server supports it 2796eaa1d2fSSunitha Harish // Else close the connection 2806eaa1d2fSSunitha Harish BMCWEB_LOG_DEBUG << "recvMessage() keepalive : " 2816eaa1d2fSSunitha Harish << self->parser->keep_alive(); 2826eaa1d2fSSunitha Harish 283039a47e3SCarson Labrado // Copy the response into a Response object so that it can be 284039a47e3SCarson Labrado // processed by the callback function. 285039a47e3SCarson Labrado self->res.clear(); 286039a47e3SCarson Labrado self->res.stringResponse = self->parser->release(); 287002d39b4SEd Tanous self->callback(self->parser->keep_alive(), self->connId, self->res); 288bd030d0aSAppaRao Puli }); 289bd030d0aSAppaRao Puli } 290bd030d0aSAppaRao Puli 2916eaa1d2fSSunitha Harish void waitAndRetry() 292bd030d0aSAppaRao Puli { 293f52c03c1SCarson Labrado if (retryCount >= retryPolicy.maxRetryAttempts) 2942a5689a7SAppaRao Puli { 2956eaa1d2fSSunitha Harish BMCWEB_LOG_ERROR << "Maximum number of retries reached."; 296f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Retry policy: " 297f52c03c1SCarson Labrado << retryPolicy.retryPolicyAction; 298039a47e3SCarson Labrado 299039a47e3SCarson Labrado // We want to return a 502 to indicate there was an error with the 300039a47e3SCarson Labrado // external server 301039a47e3SCarson Labrado res.clear(); 30240d799e6SEd Tanous res.result(boost::beast::http::status::bad_gateway); 303039a47e3SCarson Labrado 304f52c03c1SCarson Labrado if (retryPolicy.retryPolicyAction == "TerminateAfterRetries") 305fe44eb0bSAyushi Smriti { 306fe44eb0bSAyushi Smriti // TODO: delete subscription 307fe44eb0bSAyushi Smriti state = ConnState::terminated; 308039a47e3SCarson Labrado callback(false, connId, res); 309fe44eb0bSAyushi Smriti } 310f52c03c1SCarson Labrado if (retryPolicy.retryPolicyAction == "SuspendRetries") 311fe44eb0bSAyushi Smriti { 3122a5689a7SAppaRao Puli state = ConnState::suspended; 313039a47e3SCarson Labrado callback(false, connId, res); 3142a5689a7SAppaRao Puli } 3156eaa1d2fSSunitha Harish // Reset the retrycount to zero so that client can try connecting 3166eaa1d2fSSunitha Harish // again if needed 317fe44eb0bSAyushi Smriti retryCount = 0; 3182a5689a7SAppaRao Puli return; 3192a5689a7SAppaRao Puli } 3202a5689a7SAppaRao Puli 321fe44eb0bSAyushi Smriti if (runningTimer) 322fe44eb0bSAyushi Smriti { 323fe44eb0bSAyushi Smriti BMCWEB_LOG_DEBUG << "Retry timer is already running."; 324fe44eb0bSAyushi Smriti return; 325fe44eb0bSAyushi Smriti } 326fe44eb0bSAyushi Smriti runningTimer = true; 327fe44eb0bSAyushi Smriti 3282a5689a7SAppaRao Puli retryCount++; 329fe44eb0bSAyushi Smriti 330f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Attempt retry after " 331f52c03c1SCarson Labrado << std::to_string( 332f52c03c1SCarson Labrado retryPolicy.retryIntervalSecs.count()) 333fe44eb0bSAyushi Smriti << " seconds. RetryCount = " << retryCount; 334f52c03c1SCarson Labrado timer.expires_after(retryPolicy.retryIntervalSecs); 335cb13a392SEd Tanous timer.async_wait( 336f52c03c1SCarson Labrado [self(shared_from_this())](const boost::system::error_code ec) { 3376eaa1d2fSSunitha Harish if (ec == boost::asio::error::operation_aborted) 3386eaa1d2fSSunitha Harish { 3396eaa1d2fSSunitha Harish BMCWEB_LOG_DEBUG 3406eaa1d2fSSunitha Harish << "async_wait failed since the operation is aborted" 3416eaa1d2fSSunitha Harish << ec.message(); 3426eaa1d2fSSunitha Harish } 3436eaa1d2fSSunitha Harish else if (ec) 3446eaa1d2fSSunitha Harish { 3456eaa1d2fSSunitha Harish BMCWEB_LOG_ERROR << "async_wait failed: " << ec.message(); 3466eaa1d2fSSunitha Harish // Ignore the error and continue the retry loop to attempt 3476eaa1d2fSSunitha Harish // sending the event as per the retry policy 3486eaa1d2fSSunitha Harish } 349fe44eb0bSAyushi Smriti self->runningTimer = false; 3506eaa1d2fSSunitha Harish 351f52c03c1SCarson Labrado // Let's close the connection and restart from resolve. 352f52c03c1SCarson Labrado self->doCloseAndRetry(); 353fe44eb0bSAyushi Smriti }); 3542a5689a7SAppaRao Puli } 3552a5689a7SAppaRao Puli 356f52c03c1SCarson Labrado void doClose() 357fe44eb0bSAyushi Smriti { 358f52c03c1SCarson Labrado state = ConnState::closeInProgress; 359f52c03c1SCarson Labrado boost::beast::error_code ec; 360f52c03c1SCarson Labrado conn.socket().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); 361f52c03c1SCarson Labrado conn.close(); 362f52c03c1SCarson Labrado 363f52c03c1SCarson Labrado // not_connected happens sometimes so don't bother reporting it. 364f52c03c1SCarson Labrado if (ec && ec != boost::beast::errc::not_connected) 3652a5689a7SAppaRao Puli { 366f52c03c1SCarson Labrado BMCWEB_LOG_ERROR << host << ":" << std::to_string(port) 367f52c03c1SCarson Labrado << ", id: " << std::to_string(connId) 368f52c03c1SCarson Labrado << "shutdown failed: " << ec.message(); 3696eaa1d2fSSunitha Harish return; 3706eaa1d2fSSunitha Harish } 371f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << host << ":" << std::to_string(port) 372f52c03c1SCarson Labrado << ", id: " << std::to_string(connId) 373f52c03c1SCarson Labrado << " closed gracefully"; 374ca723762SEd Tanous 375f52c03c1SCarson Labrado state = ConnState::closed; 3766eaa1d2fSSunitha Harish } 377f52c03c1SCarson Labrado 378f52c03c1SCarson Labrado void doCloseAndRetry() 37992a74e56SAppaRao Puli { 380f52c03c1SCarson Labrado state = ConnState::closeInProgress; 381f52c03c1SCarson Labrado boost::beast::error_code ec; 382f52c03c1SCarson Labrado conn.socket().shutdown(boost::asio::ip::tcp::socket::shutdown_both, ec); 383f52c03c1SCarson Labrado conn.close(); 384f52c03c1SCarson Labrado 385f52c03c1SCarson Labrado // not_connected happens sometimes so don't bother reporting it. 386f52c03c1SCarson Labrado if (ec && ec != boost::beast::errc::not_connected) 38792a74e56SAppaRao Puli { 388f52c03c1SCarson Labrado BMCWEB_LOG_ERROR << host << ":" << std::to_string(port) 389f52c03c1SCarson Labrado << ", id: " << std::to_string(connId) 390f52c03c1SCarson Labrado << "shutdown failed: " << ec.message(); 3916eaa1d2fSSunitha Harish return; 3926eaa1d2fSSunitha Harish } 393f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << host << ":" << std::to_string(port) 394f52c03c1SCarson Labrado << ", id: " << std::to_string(connId) 395f52c03c1SCarson Labrado << " closed gracefully"; 396ca723762SEd Tanous 397f52c03c1SCarson Labrado // Now let's try to resend the data 398f52c03c1SCarson Labrado state = ConnState::retry; 399ca723762SEd Tanous doResolve(); 400bd030d0aSAppaRao Puli } 401bd030d0aSAppaRao Puli 402bd030d0aSAppaRao Puli public: 4038a592810SEd Tanous explicit ConnectionInfo(boost::asio::io_context& ioc, 4048a592810SEd Tanous const std::string& idIn, const std::string& destIP, 4058a592810SEd Tanous const uint16_t destPort, 4068a592810SEd Tanous const unsigned int connIdIn) : 4078a592810SEd Tanous subId(idIn), 4088a592810SEd Tanous host(destIP), port(destPort), connId(connIdIn), conn(ioc), timer(ioc) 409244256ccSCarson Labrado {} 410f52c03c1SCarson Labrado }; 411bd030d0aSAppaRao Puli 412f52c03c1SCarson Labrado class ConnectionPool : public std::enable_shared_from_this<ConnectionPool> 413bd030d0aSAppaRao Puli { 414f52c03c1SCarson Labrado private: 415f52c03c1SCarson Labrado boost::asio::io_context& ioc; 416f52c03c1SCarson Labrado const std::string id; 417f52c03c1SCarson Labrado const std::string destIP; 418f52c03c1SCarson Labrado const uint16_t destPort; 419f52c03c1SCarson Labrado std::vector<std::shared_ptr<ConnectionInfo>> connections; 420f52c03c1SCarson Labrado boost::container::devector<PendingRequest> requestQueue; 421f52c03c1SCarson Labrado 422f52c03c1SCarson Labrado friend class HttpClient; 423f52c03c1SCarson Labrado 424244256ccSCarson Labrado // Configure a connections's request, callback, and retry info in 425244256ccSCarson Labrado // preparation to begin sending the request 426f52c03c1SCarson Labrado void setConnProps(ConnectionInfo& conn) 427bd030d0aSAppaRao Puli { 428f52c03c1SCarson Labrado if (requestQueue.empty()) 429f52c03c1SCarson Labrado { 430f52c03c1SCarson Labrado BMCWEB_LOG_ERROR 431f52c03c1SCarson Labrado << "setConnProps() should not have been called when requestQueue is empty"; 432bd030d0aSAppaRao Puli return; 433bd030d0aSAppaRao Puli } 434bd030d0aSAppaRao Puli 435244256ccSCarson Labrado auto nextReq = requestQueue.front(); 436244256ccSCarson Labrado conn.retryPolicy = std::move(nextReq.retryPolicy); 437244256ccSCarson Labrado conn.req = std::move(nextReq.req); 438244256ccSCarson Labrado conn.callback = std::move(nextReq.callback); 439f52c03c1SCarson Labrado 440f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Setting properties for connection " << conn.host 441f52c03c1SCarson Labrado << ":" << std::to_string(conn.port) 442a7a80296SCarson Labrado << ", id: " << std::to_string(conn.connId); 443f52c03c1SCarson Labrado 444f52c03c1SCarson Labrado // We can remove the request from the queue at this point 445f52c03c1SCarson Labrado requestQueue.pop_front(); 446f52c03c1SCarson Labrado } 447f52c03c1SCarson Labrado 448f52c03c1SCarson Labrado // Configures a connection to use the specific retry policy. 449f52c03c1SCarson Labrado inline void setConnRetryPolicy(ConnectionInfo& conn, 450f52c03c1SCarson Labrado const RetryPolicyData& retryPolicy) 4512a5689a7SAppaRao Puli { 452f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << destIP << ":" << std::to_string(destPort) 453a7a80296SCarson Labrado << ", id: " << std::to_string(conn.connId); 454f52c03c1SCarson Labrado 455f52c03c1SCarson Labrado conn.retryPolicy = retryPolicy; 456f52c03c1SCarson Labrado } 457f52c03c1SCarson Labrado 458f52c03c1SCarson Labrado // Gets called as part of callback after request is sent 459f52c03c1SCarson Labrado // Reuses the connection if there are any requests waiting to be sent 460f52c03c1SCarson Labrado // Otherwise closes the connection if it is not a keep-alive 461f52c03c1SCarson Labrado void sendNext(bool keepAlive, uint32_t connId) 462f52c03c1SCarson Labrado { 463f52c03c1SCarson Labrado auto conn = connections[connId]; 464f52c03c1SCarson Labrado // Reuse the connection to send the next request in the queue 465f52c03c1SCarson Labrado if (!requestQueue.empty()) 466f52c03c1SCarson Labrado { 467f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << std::to_string(requestQueue.size()) 468f52c03c1SCarson Labrado << " requests remaining in queue for " << destIP 469f52c03c1SCarson Labrado << ":" << std::to_string(destPort) 470f52c03c1SCarson Labrado << ", reusing connnection " 471f52c03c1SCarson Labrado << std::to_string(connId); 472f52c03c1SCarson Labrado 473f52c03c1SCarson Labrado setConnProps(*conn); 474f52c03c1SCarson Labrado 475f52c03c1SCarson Labrado if (keepAlive) 476f52c03c1SCarson Labrado { 477f52c03c1SCarson Labrado conn->sendMessage(); 4782a5689a7SAppaRao Puli } 4792a5689a7SAppaRao Puli else 4802a5689a7SAppaRao Puli { 481f52c03c1SCarson Labrado // Server is not keep-alive enabled so we need to close the 482f52c03c1SCarson Labrado // connection and then start over from resolve 483f52c03c1SCarson Labrado conn->doClose(); 484f52c03c1SCarson Labrado conn->doResolve(); 485f52c03c1SCarson Labrado } 486f52c03c1SCarson Labrado return; 487f52c03c1SCarson Labrado } 488f52c03c1SCarson Labrado 489f52c03c1SCarson Labrado // No more messages to send so close the connection if necessary 490f52c03c1SCarson Labrado if (keepAlive) 491f52c03c1SCarson Labrado { 492f52c03c1SCarson Labrado conn->state = ConnState::idle; 493f52c03c1SCarson Labrado } 494f52c03c1SCarson Labrado else 495f52c03c1SCarson Labrado { 496f52c03c1SCarson Labrado // Abort the connection since server is not keep-alive enabled 497f52c03c1SCarson Labrado conn->state = ConnState::abortConnection; 498f52c03c1SCarson Labrado conn->doClose(); 4992a5689a7SAppaRao Puli } 500bd030d0aSAppaRao Puli } 501bd030d0aSAppaRao Puli 502244256ccSCarson Labrado void sendData(std::string& data, const std::string& destUri, 503244256ccSCarson Labrado const boost::beast::http::fields& httpHeader, 504244256ccSCarson Labrado const boost::beast::http::verb verb, 505244256ccSCarson Labrado const RetryPolicyData& retryPolicy, 5066b3db60dSEd Tanous const std::function<void(Response&)>& resHandler) 507fe44eb0bSAyushi Smriti { 508f52c03c1SCarson Labrado std::weak_ptr<ConnectionPool> weakSelf = weak_from_this(); 509f52c03c1SCarson Labrado 510f52c03c1SCarson Labrado // Callback to be called once the request has been sent 511039a47e3SCarson Labrado auto cb = [weakSelf, resHandler](bool keepAlive, uint32_t connId, 512039a47e3SCarson Labrado Response& res) { 513039a47e3SCarson Labrado // Allow provided callback to perform additional processing of the 514039a47e3SCarson Labrado // request 515039a47e3SCarson Labrado resHandler(res); 516039a47e3SCarson Labrado 517f52c03c1SCarson Labrado // If requests remain in the queue then we want to reuse this 518f52c03c1SCarson Labrado // connection to send the next request 519f52c03c1SCarson Labrado std::shared_ptr<ConnectionPool> self = weakSelf.lock(); 520f52c03c1SCarson Labrado if (!self) 521f52c03c1SCarson Labrado { 522f52c03c1SCarson Labrado BMCWEB_LOG_CRITICAL << self << " Failed to capture connection"; 523f52c03c1SCarson Labrado return; 524fe44eb0bSAyushi Smriti } 525fe44eb0bSAyushi Smriti 526f52c03c1SCarson Labrado self->sendNext(keepAlive, connId); 527f52c03c1SCarson Labrado }; 528f52c03c1SCarson Labrado 529244256ccSCarson Labrado // Construct the request to be sent 530244256ccSCarson Labrado boost::beast::http::request<boost::beast::http::string_body> thisReq( 531244256ccSCarson Labrado verb, destUri, 11, "", httpHeader); 532244256ccSCarson Labrado thisReq.set(boost::beast::http::field::host, destIP); 533244256ccSCarson Labrado thisReq.keep_alive(true); 534244256ccSCarson Labrado thisReq.body() = std::move(data); 535244256ccSCarson Labrado thisReq.prepare_payload(); 536244256ccSCarson Labrado 537f52c03c1SCarson Labrado // Reuse an existing connection if one is available 538f52c03c1SCarson Labrado for (unsigned int i = 0; i < connections.size(); i++) 539fe44eb0bSAyushi Smriti { 540f52c03c1SCarson Labrado auto conn = connections[i]; 541f52c03c1SCarson Labrado if ((conn->state == ConnState::idle) || 542f52c03c1SCarson Labrado (conn->state == ConnState::initialized) || 543f52c03c1SCarson Labrado (conn->state == ConnState::closed)) 544f52c03c1SCarson Labrado { 545244256ccSCarson Labrado conn->req = std::move(thisReq); 546f52c03c1SCarson Labrado conn->callback = std::move(cb); 547f52c03c1SCarson Labrado setConnRetryPolicy(*conn, retryPolicy); 548f52c03c1SCarson Labrado std::string commonMsg = std::to_string(i) + " from pool " + 549f52c03c1SCarson Labrado destIP + ":" + std::to_string(destPort); 550f52c03c1SCarson Labrado 551f52c03c1SCarson Labrado if (conn->state == ConnState::idle) 552f52c03c1SCarson Labrado { 553f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Grabbing idle connection " 554f52c03c1SCarson Labrado << commonMsg; 555f52c03c1SCarson Labrado conn->sendMessage(); 556f52c03c1SCarson Labrado } 557f52c03c1SCarson Labrado else 558f52c03c1SCarson Labrado { 559f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Reusing existing connection " 560f52c03c1SCarson Labrado << commonMsg; 561f52c03c1SCarson Labrado conn->doResolve(); 562f52c03c1SCarson Labrado } 563f52c03c1SCarson Labrado return; 564f52c03c1SCarson Labrado } 565f52c03c1SCarson Labrado } 566f52c03c1SCarson Labrado 567f52c03c1SCarson Labrado // All connections in use so create a new connection or add request to 568f52c03c1SCarson Labrado // the queue 569f52c03c1SCarson Labrado if (connections.size() < maxPoolSize) 570f52c03c1SCarson Labrado { 571f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Adding new connection to pool " << destIP 572f52c03c1SCarson Labrado << ":" << std::to_string(destPort); 573f52c03c1SCarson Labrado auto conn = addConnection(); 574244256ccSCarson Labrado conn->req = std::move(thisReq); 575f52c03c1SCarson Labrado conn->callback = std::move(cb); 576f52c03c1SCarson Labrado setConnRetryPolicy(*conn, retryPolicy); 577f52c03c1SCarson Labrado conn->doResolve(); 578f52c03c1SCarson Labrado } 579f52c03c1SCarson Labrado else if (requestQueue.size() < maxRequestQueueSize) 580f52c03c1SCarson Labrado { 581f52c03c1SCarson Labrado BMCWEB_LOG_ERROR << "Max pool size reached. Adding data to queue."; 582244256ccSCarson Labrado requestQueue.emplace_back(std::move(thisReq), std::move(cb), 583f52c03c1SCarson Labrado retryPolicy); 584f52c03c1SCarson Labrado } 585f52c03c1SCarson Labrado else 586f52c03c1SCarson Labrado { 587f52c03c1SCarson Labrado BMCWEB_LOG_ERROR << destIP << ":" << std::to_string(destPort) 588f52c03c1SCarson Labrado << " request queue full. Dropping request."; 589f52c03c1SCarson Labrado } 590f52c03c1SCarson Labrado } 591f52c03c1SCarson Labrado 592f52c03c1SCarson Labrado std::shared_ptr<ConnectionInfo>& addConnection() 593f52c03c1SCarson Labrado { 594f52c03c1SCarson Labrado unsigned int newId = static_cast<unsigned int>(connections.size()); 595f52c03c1SCarson Labrado 596244256ccSCarson Labrado auto& ret = connections.emplace_back( 597244256ccSCarson Labrado std::make_shared<ConnectionInfo>(ioc, id, destIP, destPort, newId)); 598f52c03c1SCarson Labrado 599f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Added connection " 600f52c03c1SCarson Labrado << std::to_string(connections.size() - 1) 601f52c03c1SCarson Labrado << " to pool " << destIP << ":" 602f52c03c1SCarson Labrado << std::to_string(destPort); 603f52c03c1SCarson Labrado 604f52c03c1SCarson Labrado return ret; 605f52c03c1SCarson Labrado } 606f52c03c1SCarson Labrado 607f52c03c1SCarson Labrado public: 6088a592810SEd Tanous explicit ConnectionPool(boost::asio::io_context& iocIn, 6098a592810SEd Tanous const std::string& idIn, 6108a592810SEd Tanous const std::string& destIPIn, 6118a592810SEd Tanous const uint16_t destPortIn) : 6128a592810SEd Tanous ioc(iocIn), 6138a592810SEd Tanous id(idIn), destIP(destIPIn), destPort(destPortIn) 614f52c03c1SCarson Labrado { 615f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Initializing connection pool for " << destIP << ":" 616f52c03c1SCarson Labrado << std::to_string(destPort); 617f52c03c1SCarson Labrado 618f52c03c1SCarson Labrado // Initialize the pool with a single connection 619f52c03c1SCarson Labrado addConnection(); 620fe44eb0bSAyushi Smriti } 621bd030d0aSAppaRao Puli }; 622bd030d0aSAppaRao Puli 623f52c03c1SCarson Labrado class HttpClient 624f52c03c1SCarson Labrado { 625f52c03c1SCarson Labrado private: 626f52c03c1SCarson Labrado std::unordered_map<std::string, std::shared_ptr<ConnectionPool>> 627f52c03c1SCarson Labrado connectionPools; 628f52c03c1SCarson Labrado boost::asio::io_context& ioc = 629f52c03c1SCarson Labrado crow::connections::systemBus->get_io_context(); 630f52c03c1SCarson Labrado std::unordered_map<std::string, RetryPolicyData> retryInfo; 631f52c03c1SCarson Labrado HttpClient() = default; 632f52c03c1SCarson Labrado 633039a47e3SCarson Labrado // Used as a dummy callback by sendData() in order to call 634039a47e3SCarson Labrado // sendDataWithCallback() 635*02cad96eSEd Tanous static void genericResHandler(const Response& res) 636039a47e3SCarson Labrado { 637039a47e3SCarson Labrado BMCWEB_LOG_DEBUG << "Response handled with return code: " 638039a47e3SCarson Labrado << std::to_string(res.resultInt()); 6394ee8e211SEd Tanous } 640039a47e3SCarson Labrado 641f52c03c1SCarson Labrado public: 642f52c03c1SCarson Labrado HttpClient(const HttpClient&) = delete; 643f52c03c1SCarson Labrado HttpClient& operator=(const HttpClient&) = delete; 644f52c03c1SCarson Labrado HttpClient(HttpClient&&) = delete; 645f52c03c1SCarson Labrado HttpClient& operator=(HttpClient&&) = delete; 646f52c03c1SCarson Labrado ~HttpClient() = default; 647f52c03c1SCarson Labrado 648f52c03c1SCarson Labrado static HttpClient& getInstance() 649f52c03c1SCarson Labrado { 650f52c03c1SCarson Labrado static HttpClient handler; 651f52c03c1SCarson Labrado return handler; 652f52c03c1SCarson Labrado } 653f52c03c1SCarson Labrado 654039a47e3SCarson Labrado // Send a request to destIP:destPort where additional processing of the 655039a47e3SCarson Labrado // result is not required 656f52c03c1SCarson Labrado void sendData(std::string& data, const std::string& id, 657f52c03c1SCarson Labrado const std::string& destIP, const uint16_t destPort, 658f52c03c1SCarson Labrado const std::string& destUri, 659f52c03c1SCarson Labrado const boost::beast::http::fields& httpHeader, 660244256ccSCarson Labrado const boost::beast::http::verb verb, 661244256ccSCarson Labrado const std::string& retryPolicyName) 662f52c03c1SCarson Labrado { 663*02cad96eSEd Tanous const std::function<void(const Response&)> cb = genericResHandler; 664039a47e3SCarson Labrado sendDataWithCallback(data, id, destIP, destPort, destUri, httpHeader, 665244256ccSCarson Labrado verb, retryPolicyName, cb); 666039a47e3SCarson Labrado } 667039a47e3SCarson Labrado 668039a47e3SCarson Labrado // Send request to destIP:destPort and use the provided callback to 669039a47e3SCarson Labrado // handle the response 670039a47e3SCarson Labrado void sendDataWithCallback(std::string& data, const std::string& id, 671039a47e3SCarson Labrado const std::string& destIP, 672039a47e3SCarson Labrado const uint16_t destPort, 673039a47e3SCarson Labrado const std::string& destUri, 674039a47e3SCarson Labrado const boost::beast::http::fields& httpHeader, 675244256ccSCarson Labrado const boost::beast::http::verb verb, 676244256ccSCarson Labrado const std::string& retryPolicyName, 6776b3db60dSEd Tanous const std::function<void(Response&)>& resHandler) 678039a47e3SCarson Labrado { 679f52c03c1SCarson Labrado std::string clientKey = destIP + ":" + std::to_string(destPort); 680f52c03c1SCarson Labrado // Use nullptr to avoid creating a ConnectionPool each time 681f52c03c1SCarson Labrado auto result = connectionPools.try_emplace(clientKey, nullptr); 682f52c03c1SCarson Labrado if (result.second) 683f52c03c1SCarson Labrado { 684f52c03c1SCarson Labrado // Now actually create the ConnectionPool shared_ptr since it does 685f52c03c1SCarson Labrado // not already exist 686244256ccSCarson Labrado result.first->second = 687244256ccSCarson Labrado std::make_shared<ConnectionPool>(ioc, id, destIP, destPort); 688f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Created connection pool for " << clientKey; 689f52c03c1SCarson Labrado } 690f52c03c1SCarson Labrado else 691f52c03c1SCarson Labrado { 692f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Using existing connection pool for " 693f52c03c1SCarson Labrado << clientKey; 694f52c03c1SCarson Labrado } 695f52c03c1SCarson Labrado 696f52c03c1SCarson Labrado // Get the associated retry policy 697f52c03c1SCarson Labrado auto policy = retryInfo.try_emplace(retryPolicyName); 698f52c03c1SCarson Labrado if (policy.second) 699f52c03c1SCarson Labrado { 700f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "Creating retry policy \"" << retryPolicyName 701f52c03c1SCarson Labrado << "\" with default values"; 702f52c03c1SCarson Labrado } 703f52c03c1SCarson Labrado 704f52c03c1SCarson Labrado // Send the data using either the existing connection pool or the newly 705f52c03c1SCarson Labrado // created connection pool 706244256ccSCarson Labrado result.first->second->sendData(data, destUri, httpHeader, verb, 707244256ccSCarson Labrado policy.first->second, resHandler); 708f52c03c1SCarson Labrado } 709f52c03c1SCarson Labrado 710a7a80296SCarson Labrado void setRetryConfig( 711a7a80296SCarson Labrado const uint32_t retryAttempts, const uint32_t retryTimeoutInterval, 712a7a80296SCarson Labrado const std::function<boost::system::error_code(unsigned int respCode)>& 713a7a80296SCarson Labrado invalidResp, 714f52c03c1SCarson Labrado const std::string& retryPolicyName) 715f52c03c1SCarson Labrado { 716f52c03c1SCarson Labrado // We need to create the retry policy if one does not already exist for 717f52c03c1SCarson Labrado // the given retryPolicyName 718f52c03c1SCarson Labrado auto result = retryInfo.try_emplace(retryPolicyName); 719f52c03c1SCarson Labrado if (result.second) 720f52c03c1SCarson Labrado { 721f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "setRetryConfig(): Creating new retry policy \"" 722f52c03c1SCarson Labrado << retryPolicyName << "\""; 723f52c03c1SCarson Labrado } 724f52c03c1SCarson Labrado else 725f52c03c1SCarson Labrado { 726f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "setRetryConfig(): Updating retry info for \"" 727f52c03c1SCarson Labrado << retryPolicyName << "\""; 728f52c03c1SCarson Labrado } 729f52c03c1SCarson Labrado 730f52c03c1SCarson Labrado result.first->second.maxRetryAttempts = retryAttempts; 731f52c03c1SCarson Labrado result.first->second.retryIntervalSecs = 732f52c03c1SCarson Labrado std::chrono::seconds(retryTimeoutInterval); 733a7a80296SCarson Labrado result.first->second.invalidResp = invalidResp; 734f52c03c1SCarson Labrado } 735f52c03c1SCarson Labrado 736f52c03c1SCarson Labrado void setRetryPolicy(const std::string& retryPolicy, 737f52c03c1SCarson Labrado const std::string& retryPolicyName) 738f52c03c1SCarson Labrado { 739f52c03c1SCarson Labrado // We need to create the retry policy if one does not already exist for 740f52c03c1SCarson Labrado // the given retryPolicyName 741f52c03c1SCarson Labrado auto result = retryInfo.try_emplace(retryPolicyName); 742f52c03c1SCarson Labrado if (result.second) 743f52c03c1SCarson Labrado { 744f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "setRetryPolicy(): Creating new retry policy \"" 745f52c03c1SCarson Labrado << retryPolicyName << "\""; 746f52c03c1SCarson Labrado } 747f52c03c1SCarson Labrado else 748f52c03c1SCarson Labrado { 749f52c03c1SCarson Labrado BMCWEB_LOG_DEBUG << "setRetryPolicy(): Updating retry policy for \"" 750f52c03c1SCarson Labrado << retryPolicyName << "\""; 751f52c03c1SCarson Labrado } 752f52c03c1SCarson Labrado 753f52c03c1SCarson Labrado result.first->second.retryPolicyAction = retryPolicy; 754f52c03c1SCarson Labrado } 755f52c03c1SCarson Labrado }; 756bd030d0aSAppaRao Puli } // namespace crow 757