SplitFile.cpp

Go to the documentation of this file.

#include "stdafx.h"
#include "SplitFile.h"
#include "icputils.h"

const size_t SplitFile::m_bytes_per_file = 512*1024;
const size_t SplitFile::m_files_per_sub_dir = 1000;

SplitFile::SplitFile() : m_file(NULL), m_count(0), m_shflag(_SH_DENYNO), m_toggle_readonly(false), m_current_bytes(0), m_mode(IFile::ReadOnly)
{
        setLoggerName("SplitFile");
}

SplitFile::~SplitFile()
{
        close();
}

int SplitFile::open(const std::string& path, Mode mode, int shflag, bool toggle_readonly)
{
        m_toggle_readonly = toggle_readonly;
        m_shflag = shflag;
        m_mode = mode;
        m_top_directory = Poco::Path::forDirectory(path);
        m_count = 0;
        return open();
}

int SplitFile::open()
{
        m_current_bytes = 0;
        std::string file_name = getFileName();
        Poco::Path p(file_name);
        p.setFileName("");
        Poco::File f(p);
        f.createDirectories();
        if (m_toggle_readonly && (_access(file_name.c_str(), 0) == 0) )
        {
                RetryWIN32APIBOOL(boost::bind(SetFileAttributes, file_name.c_str(), FILE_ATTRIBUTE_NORMAL), 10); // clear read-only
        }
        std::string fmode = openMode(file_name);
        for(int nretry = 10; ((m_file = _fsopen(file_name.c_str(), fmode.c_str(), m_shflag)) == NULL) && (nretry > 0); --nretry)
        {
                LOGSTR_ERROR(file_name + " in mode " + fmode + ": " + strerror(errno));
                Sleep(100);
        }
        if (m_file != NULL)
        {
                return 0;
        }
        else
        {
                return -1;
        }
}

size_t SplitFile::write(const void* data, size_t element_size, size_t nelement)
{
        if (m_file == NULL)
        {
                return static_cast<size_t>(-1);  
        }
        size_t n = m_current_bytes + nelement*element_size;
        if (n <= m_bytes_per_file)
        {
                m_current_bytes = n;
                return fwrite(data, element_size, nelement, m_file);
        }
        size_t n1 = (m_bytes_per_file - m_current_bytes) / element_size;
        if (fwrite(data, element_size, n1, m_file) != n1)
        {
                return static_cast<size_t>(-1);
        }
        nextFile();
        n = write((const char*)data + n1 * element_size, element_size, nelement - n1);
        if (n != -1)
        {
                return n1 + n;
        }
        else
        {
                return static_cast<size_t>(-1);  
        }
}

size_t SplitFile::read(void* data, size_t element_size, size_t nelement)
{
        if (m_file == NULL)
        {
                return static_cast<size_t>(-1);  
        }
        if (nelement == 0)
        {
                return 0;
        }
        size_t n = m_current_bytes + nelement*element_size;
        if (n <= m_bytes_per_file)
        {
                m_current_bytes = n;
                return fread(data, element_size, nelement, m_file);
        }
        size_t n1 = (m_bytes_per_file - m_current_bytes) / element_size;
        if ( (n = fread(data, element_size, n1, m_file)) != n1 )
        {
                if (n != static_cast<size_t>(-1))
                {
                        m_current_bytes += n * element_size;
                }
                return n;
        }
        nextFile();
        n = read((char*)data + n1 * element_size, element_size, nelement - n1);
        if ( n != static_cast<size_t>(-1) )
        {
                return n1 + n;
        }
        else
        {
                return static_cast<size_t>(-1);
        }
}

void SplitFile::nextFile()
{
        if (close() != 0)
        {
                throw std::runtime_error("close");
        }
        ++m_count;
        if (open() != 0)
        {
                throw std::runtime_error("open");
        }
}

int SplitFile::close()
{
        int ret = 0;
        if (m_file != NULL)
        {
                if (flush() != 0)
                {
                        throw std::runtime_error("flush");
                }
                ret = fclose(m_file);
                m_file = NULL;
                if (m_toggle_readonly)
                {
                        RetryWIN32APIBOOL(boost::bind(SetFileAttributes, getFileName().c_str(), FILE_ATTRIBUTE_READONLY), 10);  // set read-only file attribute
                }
        }
        return ret;
}

int SplitFile::seek(int64_t offset, int whence)
{
        if (whence != SEEK_SET)
        {
                throw std::runtime_error("SplitFile::seek");
        }
        size_t count = countFromOffset(offset);
        if (count != m_count || m_file == NULL)
        {
                if (close() != 0)
                {
                        throw std::runtime_error("close");
                }
                m_count = count;
                if (open() != 0)
                {
                        throw std::runtime_error("open");
                }
        }
//      LOGSTR_INFORMATION("Seeking to " << offset << " bytes which is count=" << m_count << " offset=" << offset % m_bytes_per_file);
        m_current_bytes = offset % m_bytes_per_file;
        return fseek(m_file, m_current_bytes, whence);
}

int SplitFile::tell(int64_t& offset)
{
        offset = static_cast<uint64_t>(m_count) * m_bytes_per_file + m_current_bytes;
        if (ftell(m_file) != m_current_bytes)
        {
                throw std::runtime_error("ftell");
        }
        return 0;
}

int SplitFile::discardContents()
{
        std::set<std::string> files;
        findFiles(files);
        std::string current_file = getFileName();
        BOOST_FOREACH(const std::string& s, files)
        {
                if (s != current_file)
                {
                        RetryWIN32APIBOOL(boost::bind(SetFileAttributes, s.c_str(), FILE_ATTRIBUTE_NORMAL), 10); // clear read-only file attribute
                        std::remove(s.c_str());
                }
        }
        if (m_file != NULL)
        {
                return _chsize(fileno(m_file), 0);
        }
        else
        {
                std::remove(current_file.c_str());
                return 0;
        }
}

void SplitFile::removeFile(const std::string& path)
{
        std::set<std::string> files;
        findFiles(Poco::Path::forDirectory(path), files);
        BOOST_FOREACH(const std::string& s, files)
        {
                RetryWIN32APIBOOL(boost::bind(SetFileAttributes, s.c_str(), FILE_ATTRIBUTE_NORMAL), 10); // clear read-only file attribute
                std::remove(s.c_str());
        }
        // remove directories
        Poco::File f(path);
        try
        {
                f.remove(true);
        }
        catch(const std::exception& )
        {
                ; // LOGSTR_INFORMATION("Cannot delete " + path + ": " + ex.what());
        }
}

bool SplitFile::isOpen() const
{
        return m_file != NULL;
}

int SplitFile::flush()
{
        if (m_file != NULL)
        {
                return fflush(m_file);
        }
        else
        {
                return 0;
        }
}

size_t SplitFile::countFromOffset(int64_t offset) const
{
        return offset / m_bytes_per_file;
}

std::string SplitFile::getFileName() const
{
        return fileNamefromCount(m_count);
}

std::string SplitFile::fileNamefromCount(size_t count) const
{
        Poco::Path p(m_top_directory, Poco::format("F%lu.tmp", (unsigned long)(count % m_files_per_sub_dir)));
        p.pushDirectory(Poco::format("D%lu", (unsigned long)(count / m_files_per_sub_dir)));
        return p.toString();
}

void SplitFile::findFiles(std::set<std::string>& files) const
{
        findFiles(m_top_directory, files);
}

void SplitFile::findFiles(const Poco::Path& path, std::set<std::string>& files)
{
        files.clear();
        std::string file_expr("F[0-9]*.tmp");   // poco glob is shell like not RE like
        Poco::Path search_expr(path, file_expr);
        search_expr.pushDirectory("D[0-9]*");  // poco glob is shell like not RE like
        Poco::Glob::glob(search_expr, files, Poco::Glob::GLOB_CASELESS);
}

std::string SplitFile::openMode(const std::string& path) const
{
        std::string fmode;
        // "N" is to mark the handle as noinherit so it does not get propagated to spawned processes
        if (m_mode == IFile::ReadOnly)
        {
                fmode = "rbN";
        }
        else if (m_mode == IFile::ReadWriteCreate)
        {
                if (_access(path.c_str(), 0) == 0)
                {
                        fmode = "r+bcN";
                }
                else
                {
                        fmode = "wbcN";
                }
        }
        return fmode;
}