Belle II Software  release-05-02-19
Downloader.h
1 /**************************************************************************
2  * BASF2 (Belle Analysis Framework 2) *
3  * Copyright(C) 2016-2019 - Belle II Collaboration *
4  * *
5  * Author: The Belle II Collaboration *
6  * Contributors: Martin Ritter *
7  * *
8  * This software is provided "as is" without any warranty. *
9  **************************************************************************/
10 
11 #pragma once
12 
13 #include <framework/utilities/ScopeGuard.h>
14 
15 #include <iosfwd>
16 #include <memory>
17 
18 namespace Belle2::Conditions {
20  struct CurlSession;
21 
23  class Downloader final {
24  public:
27  Downloader() = default;
29  ~Downloader();
30 
39  bool startSession();
41  void finishSession();
45  ScopeGuard ensureSession()
46  {
47  bool started = startSession();
48  return ScopeGuard([this, started] {if (started) finishSession();});
49  }
50 
52  unsigned int getConnectionTimeout() const { return m_connectionTimeout; }
54  unsigned int getStalledTimeout() const { return m_stalledTimeout; }
56  unsigned int getMaxRetries() const { return m_maxRetries; }
58  unsigned int getBackoffFactor() const { return m_backoffFactor; }
60  void setConnectionTimeout(unsigned int timeout);
62  void setStalledTimeout(unsigned int timeout);
64  void setMaxRetries(unsigned int retries) { m_maxRetries = retries; }
66  void setBackoffFactor(unsigned int factor) { m_backoffFactor = std::max(1u, factor); }
67 
77  bool download(const std::string& url, std::ostream& stream, bool silentOnMissing = false);
78 
84  bool verifyChecksum(std::istream& input, const std::string& checksum) { return calculateChecksum(input) == checksum; }
86  std::string escapeString(const std::string& text);
88  std::string joinWithSlash(const std::string& base, const std::string& second);
89 
90  private:
95  static std::string calculateChecksum(std::istream& input);
96 
98  std::unique_ptr<CurlSession> m_session;
100  static bool s_globalInit;
102  unsigned int m_connectionTimeout{60};
104  unsigned int m_stalledTimeout{60};
106  unsigned int m_maxRetries{5};
108  unsigned int m_backoffFactor{5};
109  };
110 } // namespace Belle2::Conditions
Belle2::Conditions::Downloader::getBackoffFactor
unsigned int getBackoffFactor() const
Get the backoff factor for retries in seconds.
Definition: Downloader.h:66
Belle2::Conditions::Downloader::setMaxRetries
void setMaxRetries(unsigned int retries)
Set the number of retries to perform when downloading fails with HTTP response code >=500,...
Definition: Downloader.h:72
Belle2::Conditions::Downloader::finishSession
void finishSession()
Finish an existing curl session if any is active at the moment.
Definition: Downloader.cc:224
Belle2::Conditions::Downloader::download
bool download(const std::string &url, std::ostream &stream, bool silentOnMissing=false)
get an url and save the content to stream This function raises exceptions when there are any problems
Definition: Downloader.cc:268
Belle2::Conditions::Downloader::calculateChecksum
static std::string calculateChecksum(std::istream &input)
calculate the digest/checksum on a given string.
Definition: Downloader.cc:234
Belle2::Conditions::Downloader::~Downloader
~Downloader()
Destructor.
Definition: Downloader.cc:149
Belle2::Conditions::Downloader::setStalledTimeout
void setStalledTimeout(unsigned int timeout)
Set the timeout to wait for stalled connections (<10KB/s), 0 disables timeout.
Definition: Downloader.cc:260
Belle2::Conditions::Downloader::ensureSession
ScopeGuard ensureSession()
Make sure there's an active session and return a ScopeGuard object that closes the session on destruc...
Definition: Downloader.h:53
Belle2::Conditions::Downloader::m_stalledTimeout
unsigned int m_stalledTimeout
Timeout to wait for stalled connections (<10KB/s)
Definition: Downloader.h:112
Belle2::Conditions::Downloader::escapeString
std::string escapeString(const std::string &text)
Escape a string to make it safe to be used in web requests.
Definition: Downloader.cc:151
Belle2::Conditions::Downloader::joinWithSlash
std::string joinWithSlash(const std::string &base, const std::string &second)
Join two strings and make sure that there is exactly one '/' between them.
Definition: Downloader.cc:165
Belle2::Conditions::Downloader::Downloader
Downloader()=default
Create a new payload downloader.
Belle2::Conditions::Downloader::m_maxRetries
unsigned int m_maxRetries
Number of retries to perform when downloading fails with HTTP response code >=500.
Definition: Downloader.h:114
Belle2::Conditions::Downloader::m_connectionTimeout
unsigned int m_connectionTimeout
Timeout to wait for connections in seconds.
Definition: Downloader.h:110
Belle2::Conditions::Downloader::s_globalInit
static bool s_globalInit
flag to indicate whether curl has been initialized already
Definition: Downloader.h:108
Belle2::Conditions::Downloader::getStalledTimeout
unsigned int getStalledTimeout() const
Get the timeout to wait for stalled connections (<10KB/s), 0 means no timeout.
Definition: Downloader.h:62
Belle2::Conditions::Downloader::startSession
bool startSession()
Start a new curl session if none is active at the moment.
Definition: Downloader.cc:171
Belle2::Conditions::Downloader::getDefaultInstance
static Downloader & getDefaultInstance()
Return the default instance.
Definition: Downloader.cc:143
Belle2::Conditions::Downloader::getMaxRetries
unsigned int getMaxRetries() const
Get the number of retries to perform when downloading fails with HTTP response code >=500,...
Definition: Downloader.h:64
Belle2::Conditions::Downloader::setConnectionTimeout
void setConnectionTimeout(unsigned int timeout)
Set the timeout to wait for connections in seconds, 0 means built in curl default.
Definition: Downloader.cc:252
Belle2::Conditions::Downloader::getConnectionTimeout
unsigned int getConnectionTimeout() const
Get the timeout to wait for connections in seconds, 0 means the built in curl default.
Definition: Downloader.h:60
Belle2::Conditions::Downloader::verifyChecksum
bool verifyChecksum(std::istream &input, const std::string &checksum)
check the digest of a stream
Definition: Downloader.h:92
Belle2::Conditions::Downloader::m_session
std::unique_ptr< CurlSession > m_session
curl session handle
Definition: Downloader.h:106
Belle2::Conditions::Downloader::setBackoffFactor
void setBackoffFactor(unsigned int factor)
Set the backoff factor for retries in seconds.
Definition: Downloader.h:74
Belle2::Conditions::Downloader::m_backoffFactor
unsigned int m_backoffFactor
Backoff factor for retries in seconds.
Definition: Downloader.h:116