gzx.hpp

Go to the documentation of this file.

/*
 * Copyright 2008,2009,2010,2011,2012 Stephan Ossowski, Korbinian Schneeberger,
 * Felix Ott, Joerg Hagmann, Alf Scotland, Sebastian Bender
 * 
 * This file is part of SHORE.
 * 
 * SHORE is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * SHORE is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with SHORE.  If not, see <http://www.gnu.org/licenses/>.
 */


#ifndef GZIPINDEX_HPP__
#define GZIPINDEX_HPP__

#include <iostream>
#include <map>
#include <vector>

#include <zlib.h>

#include <boost/iostreams/concepts.hpp>
#include <boost/iostreams/traits.hpp>

#include "shore/stream/streams.hpp"

namespace shore {

class gzxbase
{
 protected:

        int m_zstat;


        void z_error(const std::string &msg);
        void zstat_error(const std::string &msg);
};

class gzxindex
:public gzxbase
{
 public:
        
        class iterator
        :public gzxbase
        {
         private:

                friend class gzxindex;

                const gzxindex *m_parent;
                size_t m_block;


                iterator(const gzxindex *const parent);

                void init();

         public:

                iterator();
                iterator(const iterator &other);
                iterator &operator=(const iterator &other);

                void locate(const uint64_t uncompressed_pos);
                void next_block();
                void next_stream();
                void rewind();

                bool eof() const;

                uint64_t compressed_file_offset() const;
                uint64_t uncompressed_file_offset() const;
        };

        enum ChkType
        {
                CHK_CRC32=0,CHK_ALR32=1,CHK_UNKNOWN
        };

 private:

        gzxindex* m_previous;
        gzxindex* m_next;
        uint64_t m_inf_offset;
        uint64_t m_def_offset;

        uint64_t m_blocksize;
        uint64_t m_totalsize;
        uint64_t m_deflatedsize;
        ChkType m_chktype;
        std::vector<uint64_t> m_syncofs;
        std::vector<uint64_t> m_chksums;


        struct mdparser
        {
                gzxindex*const that;
                bool fmtinit;
 
                mdparser(gzxindex*const t):that(t),fmtinit(false) {}
                void operator()(const std::string& md);
        }
        m_mdparse;


        static bool get_gzxblock(std::string& buf,std::istream& is);

        void parse(std::istream& in);
        bool read_record(std::istream& is,const std::streampos endofs=0);


        gzxindex(gzxindex* nxt);

        void parse_old(std::istream& in);

 public:

        gzxindex(const uint64_t blocksize,const ChkType check);
        gzxindex(std::istream *const is);
        ~gzxindex();

        void append(uint64_t syncoff,uint64_t chksum);
        void set_totals(const uint64_t inflated,const uint64_t deflated);
        void write_record(std::ostream& os) const;

        bool valid() const;

        size_t get_stream_count() const;
        size_t get_block_count() const;
        uint64_t get_uncompressed_size() const;

        iterator begin() const;

        void print(std::ostream &os) const;

};

class gzxdeflater
:public gzxbase
{
 private:

        static const int WINDOWBITS=15;
        static const int MEMLEVEL=8;

        ::z_stream m_strm;

        std::ostream& m_os;

        size_t m_blocksize;
        bool m_syncblocks;

        uint8_t* m_inbuf;
        uint8_t* m_outbuf;

        gzxindex m_syncs;

        gzxdeflater(const gzxdeflater&);

        void deflate_block();

 public:

        enum HeaderType
        {
                HEADER_RAW=-1,
                HEADER_ZLIB=0,
                HEADER_GZIP=16
        };

        enum Strategy
        {
                STGY_DEFAULT=Z_DEFAULT_STRATEGY,
                STGY_FILTERED=Z_FILTERED,
                STGY_HUFFMAN=Z_HUFFMAN_ONLY,
                STGY_RLE=Z_RLE,
                STGY_FIXED=Z_FIXED
        };

        gzxdeflater(std::ostream& os,const bool syncblocks=true,
                    const size_t bs=131072,const HeaderType ht=HEADER_GZIP,
                    const Strategy sg=STGY_DEFAULT);

        ~gzxdeflater();

        void finish();
        
        void write(const char* s,size_t n);

        const gzxindex &get_syncs() const;
};

class gzxdeflater_sink
:public boost::iostreams::sink
{
 private:

        gzxdeflater* m_def;

 public:

        gzxdeflater_sink(gzxdeflater*const def);
        std::streamsize write(const char* s,std::streamsize n);
        void close();
};

class gzx_ostream
:public boost::iostreams::stream<gzxdeflater_sink>
{
 private:

        gzxdeflater m_def;

 public:

        gzx_ostream(std::ostream* os,const bool syncblocks=true,
                    const size_t bs=131072);

        ~gzx_ostream();
};

class gzxinflater
:public gzxbase
{
 private:

        static const int WINDOWBITS=15;
        static const size_t BUFSIZE=8192;
        uint8_t* m_inbuf;
        uint8_t* m_outbuf;
        size_t m_outofs;


        enum HeaderType
        {
                HEADER_RAW=-1,
                HEADER_ZLIB=0,
                HEADER_GZIP=16,
                HEADER_AUTO=32
        };

        ::z_stream m_strm;

        std::istream& m_is;

        gzxindex m_gzx;
        gzxindex::iterator m_gzxiter;

        uint64_t m_inflpos;

        enum
        {
                INF_STOPPED=0,
                INF_GZ=1,
                INF_RAW=2
        }
        m_inflatemode;


        gzxinflater(const gzxinflater&);

        void inflate_start(const size_t defl_pos=0);

        void inflate_stop();

 public:

        gzxinflater(std::istream& is,const bool raw_seekable=true);

        ~gzxinflater();

        std::streamsize read(char* s,size_t n);

        std::streampos seek(int64_t ofs,std::ios_base::seekdir way);

        const gzxindex &get_index() const;
};

class gzxinflater_source
:public boost::iostreams::device<boost::iostreams::input_seekable>
{
 private:

        gzxinflater* m_inf;

 public:

        gzxinflater_source(gzxinflater*const inf);

        std::streamsize read(char* s,std::streamsize n);

        std::streampos seek(boost::iostreams::stream_offset off,
                            std::ios_base::seekdir way);
};

class gzx_istream
:public boost::iostreams::stream<gzxinflater_source>
{
 private:

        gzxinflater m_inf;

 public:

        gzx_istream(std::istream* is,const bool raw_seekable=true);

        ~gzx_istream();

        const gzxindex &get_index() const;

        static std::string inflate(const std::string& zdata);
};

} // namespace shore

#endif // GZIPINDEX_HPP__