SeqFile Class Reference

A class to manage I/O for a chain of blocked files. More...

#include <SeqFile.h>

Collaboration diagram for SeqFile:

Public Member Functions
	SeqFile (const std::string &filename, const std::string &rwflag, char *streamerinfo=nullptr, int streamerinfo_size=0, bool filenameIsPattern=false)
	Constructor.
	~SeqFile ()
	Destructor.
	SeqFile (const SeqFile &)=delete
	No copying.
SeqFile &	operator= (const SeqFile &)=delete
	No assignment.
int	status () const
	Returns status after constructor call.
int	write (const char *buf)
	Write a record to a file.
int	read (char *buf, int max)
	Read a record from a file.

Private Member Functions
void	openFile (std::string filename, bool readonly)
	actually open the file

Private Attributes
std::string	m_filename
	Name of the opened file.
std::string	m_filenamePattern
	Pattern for creating the file from the sequence number.
int	m_fd { -1}
	file descriptor.
int	m_nb {0}
	when saving a file, the total number of bytes written, 0 when reading.
int	m_nfile {0}
	file counter, starting at 0 (files are split after c_MaxFileSize bytes).
bool	m_compressed {false}
	is file gzipped compressed?
std::unique_ptr< std::ios >	m_stream
	pointer to the filtering input or output stream
char *	m_streamerinfo
	StreamerInfo.
int	m_streamerinfo_size
	size(bytes) of StreamerInfo

Static Private Attributes
static const int	c_MaxFileSize {512000000 * 4}
	maximal size of one file (in Bytes).

Detailed Description

A class to manage I/O for a chain of blocked files.

Definition at line 22 of file SeqFile.h.

Constructor & Destructor Documentation

SeqFile()

SeqFile	(	const std::string &	filename,
		const std::string &	rwflag,
		char *	streamerinfo = nullptr,
		int	streamerinfo_size = 0,
		bool	filenameIsPattern = false
	)

Constructor.

Parameters

filename	name of the file
rwflag	should probably be r or rw
streamerinfo	String containing the streamer info
streamerinfo_size	Size of the string containing the streamer info
filenameIsPattern	if true interpret the filename as a boost::format pattern which takes the sequence number as argument instead of producing .sroot-N files

Definition at line 25 of file SeqFile.cc.

                                        :
  m_filename(filename)
{
  if (m_filename.empty()) {
    B2ERROR("SeqFile: Empty filename given");
    return;
  }
  bool readonly = rwflag.find('w') == std::string::npos;
  // CAF use URL input style and add a prefix that need to be removed for sroot files
  if (m_filename.compare(0, 7, "file://") == 0) {
    m_filename = m_filename.substr(7, m_filename.size() - 7);
  }
  // is the file already compressed?
  m_compressed = m_filename.size() > 3 && m_filename.compare(m_filename.size() - 3, 3, ".gz") == 0;
  // strip .gz suffix to add it at the end automatically and correctly for subsequent files
  if (m_compressed) {
    m_filename = m_filename.substr(0, m_filename.size() - 3);
  }
  // check if we want different naming scheme using boost::format
  if (filenameIsPattern) {
    m_filenamePattern = m_filename;
    try {
      m_filename = (boost::format(m_filenamePattern) % m_nfile).str();
    } catch (std::exception& e) {
      B2FATAL("Cannot use filename pattern" << m_filenamePattern << ": " << e.what());
    }
  }
 
  // Store StreamerInfo 2017.5.8
  m_streamerinfo = nullptr;
  m_streamerinfo_size = 0;
  if (streamerinfo != nullptr && streamerinfo_size > 0) {
    m_streamerinfo_size = streamerinfo_size;
    m_streamerinfo = new char[ m_streamerinfo_size ];
    memcpy(m_streamerinfo, streamerinfo, m_streamerinfo_size);
  }
 
  // open the file
  openFile(m_filename, readonly);
  // if that fails and it's not already assumed to be compressed try again adding .gz to the name
  if (m_fd < 0 && !m_compressed) {
    B2WARNING("SeqFile: error opening '" << m_filename << "': " << strerror(errno)
              << ", trying again with '.gz'");
    m_compressed = true;
    openFile(m_filename, readonly);
  }
  // is the file open now?
  if (m_fd < 0) {
    B2ERROR("SeqFile: error opening '" << m_filename << "': " << strerror(errno));
  } else {
    B2INFO("SeqFile: " << m_filename << " opened (fd=" << m_fd << ")");
  }
 
}

~SeqFile()

~SeqFile

(

)

Destructor.

Definition at line 128 of file SeqFile.cc.

{
  if (m_streamerinfo != nullptr) delete m_streamerinfo;
  B2INFO("Closing SeqFile " << m_nfile);
  //closed automatically by m_stream.
}

Member Function Documentation

openFile()

void openFile ( std::string  filename, bool  readonly  )

private

actually open the file

Definition at line 81 of file SeqFile.cc.

{
 
  // add compression suffix if file is supposed to be compressed
  if (m_compressed) filename += ".gz";
  if (!readonly) {
    //open file in create mode and set stream correctly
    m_fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0664);
    auto filter = new io::filtering_ostream();
    if (m_compressed) filter->push(io::gzip_compressor());
    filter->push(io::file_descriptor_sink(m_fd, io::close_handle));
    filter->exceptions(ios_base::badbit | ios_base::failbit);
    m_stream.reset(filter);
 
    //
    // Write StreamerInfo  (2017.5.8)
    //
    if (m_streamerinfo == nullptr || m_streamerinfo_size <= 0) {
      // If you want to use SeqFile for non-sroot file type, please skip this B2FATAL
      B2FATAL("Invalid size of StreamerInfo : " << m_streamerinfo_size << "bytes");
    } else {
      auto* out = dynamic_cast<std::ostream*>(m_stream.get());
      if (!out) {
        B2FATAL("SeqFile::write() called on a file opened in read mode");
      }
      try {
        out->write(m_streamerinfo, m_streamerinfo_size);
        B2INFO("Wrote StreamerInfo at the begenning of the file. : " << m_streamerinfo_size << "bytes");
      } catch (ios_base::failure& e) {
 
        B2ERROR("SeqFile::openFile() error: " << e.what() << ", " << strerror(errno));
      }
    }
 
  } else {
    //open file in read mode and set stream correctly
    m_fd = open(filename.c_str(), O_RDONLY);
    auto filter = new io::filtering_istream();
    if (m_compressed) filter->push(io::gzip_decompressor());
    filter->push(io::file_descriptor_source(m_fd, io::close_handle));
    filter->exceptions(ios_base::badbit | ios_base::failbit);
    m_stream.reset(filter);
  }
  // reset number of written bytes (does not include streamerinfo )
  m_nb = 0;
}

read()

int read	(	char *	buf,
		int	max
	)

Read a record from a file.

The record length is returned.

Definition at line 174 of file SeqFile.cc.

{
  // cast stream object
  auto* in = dynamic_cast<std::istream*>(m_stream.get());
  if (!in) {
    B2FATAL("SeqFile::read() called on a file opened in write mode");
  }
  //trigger eof if there's nothing left int the file. Could throw an error on decompress failure
  try {
    in->peek();
  } catch (ios_base::failure& e) {
    B2ERROR("SeqFile::read() cannot read file: " << e.what());
    return -1;
  }
  //ok, now we can open the next file reliably
  if (in->eof()) {
    // EOF of current file, search for next file
    m_nfile++;
    auto nextfile = m_filename + '-' + std::to_string(m_nfile);
    if (!m_filenamePattern.empty()) {
      nextfile = (boost::format(m_filenamePattern) % m_nfile).str();
    }
    openFile(nextfile, true);
    if (m_fd < 0) return 0;   // End of all files
    // update the stream pointer
    in = dynamic_cast<std::istream*>(m_stream.get());
    B2INFO("SeqFile::read() opened '" << nextfile << "'");
  }
  try {
    // Obtain new header
    in->read(buf, sizeof(int));
  } catch (ios_base::failure& e) {
    B2ERROR("SeqFile::read() " << e.what() << ": couldn't read next record size");
    return -1;
  }
  // Normal processing, extract the record size from the first 4 bytes
  int recsize = *((int*)buf);
  if (recsize > size) {
    B2ERROR("SeqFile::read() buffer too small, need at least " << recsize << " bytes");
    return -1;
  }
  try {
    in->read(buf + sizeof(int), recsize - sizeof(int));
  } catch (ios_base::failure& e) {
    B2ERROR("SeqFile::read() " << e.what() << ": could only read " << in->gcount() << " bytes, expected " << recsize);
    return -1;
  }
  return recsize;
}

status()

int status

(

)

const

Returns status after constructor call.

If success, fd is returned. If not, -1

Definition at line 135 of file SeqFile.cc.

{
  return m_fd;
}

write()

int write

(

const char *

buf

)

Write a record to a file.

First word of the record should contain number of words.

Definition at line 140 of file SeqFile.cc.

{
  // cast stream object
  auto* out = dynamic_cast<std::ostream*>(m_stream.get());
  if (!out) {
    B2FATAL("SeqFile::write() called on a file opened in read mode");
  }
  int insize = *((int*)buf); // nbytes in the buffer at the beginning
  if (insize + m_nb >= c_MaxFileSize && m_filename != "/dev/null") {
    B2INFO("SeqFile: previous file closed (size=" << m_nb << " bytes)");
    m_nfile++;
    auto file = m_filename + '-' + std::to_string(m_nfile);
    if (!m_filenamePattern.empty()) {
      file = (boost::format(m_filenamePattern) % m_nfile).str();
    }
    openFile(file, false);
    if (m_fd < 0) {
      B2FATAL("SeqFile::write() error opening file '" << file << "': " << strerror(errno));
    }
    m_nb = 0;
    B2INFO("SeqFile::write() opened '" << file << "'");
    // update stream pointer since we reopened the file
    out = dynamic_cast<std::ostream*>(m_stream.get());
  }
  try {
    out->write(buf, insize);
    m_nb += insize;
    return insize;
  } catch (ios_base::failure& e) {
    B2ERROR("SeqFile::write() error: " << e.what() << ", " << strerror(errno));
    return 0;
  }
}

Member Data Documentation

c_MaxFileSize

const int c_MaxFileSize {512000000 * 4}

staticprivate

maximal size of one file (in Bytes).

Definition at line 57 of file SeqFile.h.

m_compressed

bool m_compressed {false}

private

is file gzipped compressed?

Definition at line 64 of file SeqFile.h.

m_fd

int m_fd { -1}

private

file descriptor.

Definition at line 61 of file SeqFile.h.

m_filename

std::string m_filename

private

Name of the opened file.

Definition at line 59 of file SeqFile.h.

m_filenamePattern

std::string m_filenamePattern

private

Pattern for creating the file from the sequence number.

Definition at line 60 of file SeqFile.h.

m_nb

int m_nb {0}

private

when saving a file, the total number of bytes written, 0 when reading.

Definition at line 62 of file SeqFile.h.

m_nfile

int m_nfile {0}

private

file counter, starting at 0 (files are split after c_MaxFileSize bytes).

Definition at line 63 of file SeqFile.h.

m_stream

std::unique_ptr<std::ios> m_stream

private

pointer to the filtering input or output stream

Definition at line 65 of file SeqFile.h.

m_streamerinfo

char* m_streamerinfo

private

StreamerInfo.

Definition at line 68 of file SeqFile.h.

m_streamerinfo_size

int m_streamerinfo_size

private

size(bytes) of StreamerInfo

Definition at line 71 of file SeqFile.h.

---

The documentation for this class was generated from the following files:

• framework/pcore/include/SeqFile.h
• framework/pcore/src/SeqFile.cc