column.h

Go to the documentation of this file.

//File: $Id$
// Author: John Wu <John.Wu at ACM.org>
// Copyright 2000-2011 the Regents of the University of California
#ifndef IBIS_COLUMN_H
#define IBIS_COLUMN_H







#include "table.h"      // ibis::TYPE_T
#include "qExpr.h"      // ibis::qContinuousRange
#include "bitvector.h"
#include <string>

namespace ibis { // additional names to the namespace ibis
    // derived classes of ibis::column, implemented in category.cpp
    class category;   // for categorical values (low-cardinality text fields)
    class text;       // arbitrary cardinality text fields
    class blob;       // text may contain null characters

    // the following are used for storing selected values of different types
    // of columns (implemented in colValues.cpp)
    class colBytes;
    class colUBytes;
    class colShorts;
    class colUShorts;
    class colInts;
    class colUInts;
    class colLongs;
    class colULongs;
    class colFloats;
    class colDoubles;
    class colStrings;
} // namespace

class FASTBIT_CXX_DLLSPEC ibis::column {
public:

    virtual ~column();
    column(const part* tbl, FILE* file);
    column(const part* tbl, ibis::TYPE_T t, const char* name,
           const char* desc="", double low=DBL_MAX, double high=-DBL_MAX);
    column(const column& rhs); 

    ibis::TYPE_T type() const {return m_type;}
    const char* name() const {return m_name.c_str();}
    void name(const char* nm) {m_name = nm;}
    const char* description() const {return m_desc.c_str();}
    const double& lowerBound() const {return lower;}
    const double& upperBound() const {return upper;}

    int elementSize() const;
    bool isFloat() const;
    bool isInteger() const;
    bool isSignedInteger() const;
    bool isUnsignedInteger() const;
    bool isNumeric() const;
    bool isSorted() const {return m_sorted;} 
    void description(const char* d) {m_desc = d;}
    void lowerBound(double d) {lower = d;}
    void upperBound(double d) {upper = d;}
    const part* partition() const {return thePart;}
    void isSorted(bool);

    // function related to index/bin
    const char* indexSpec() const; 
    uint32_t numBins() const; 

    void indexSpec(const char* spec) {m_bins=spec;}
    void preferredBounds(std::vector<double>&) const;
    void binWeights(std::vector<uint32_t>&) const;

    virtual void computeMinMax();
    virtual void computeMinMax(const char *dir);
    virtual void computeMinMax(const char *dir,
                               double& min, double &max) const;

    virtual void loadIndex(const char* iopt=0, int ropt=0) const throw ();
    virtual void unloadIndex() const;
    virtual long indexSize() const;

    uint32_t indexedRows() const;
    void indexSpeedTest() const;
    void purgeIndexFile(const char *dir=0) const;

    const char* dataFileName(std::string& fname, const char *dir=0) const;
    const char* nullMaskName(std::string& fname) const;
    void getNullMask(bitvector& mask) const;
    int  setNullMask(const bitvector&);

    virtual void getString(uint32_t, std::string &) const {};
    virtual const char* findString(const char*) const
    {return static_cast<const char*>(0);}

    array_t<int32_t>* getIntArray() const;
    array_t<float>*   getFloatArray() const;
    array_t<double>*  getDoubleArray() const;
    virtual int getValuesArray(void* vals) const;
    virtual ibis::fileManager::storage* getRawData() const;

    virtual array_t<signed char>*   selectBytes(const bitvector& mask) const;
    virtual array_t<unsigned char>* selectUBytes(const bitvector& mask) const;
    virtual array_t<int16_t>*  selectShorts(const bitvector& mask) const;
    virtual array_t<uint16_t>* selectUShorts(const bitvector& mask) const;
    virtual array_t<int32_t>*  selectInts(const bitvector& mask) const;
    virtual array_t<uint32_t>* selectUInts(const bitvector& mask) const;
    virtual array_t<int64_t>*  selectLongs(const bitvector& mask) const;
    virtual array_t<uint64_t>* selectULongs(const bitvector& mask) const;
    virtual array_t<float>*    selectFloats(const bitvector& mask) const;
    virtual array_t<double>*   selectDoubles(const bitvector& mask) const;
    virtual std::vector<std::string>*
        selectStrings(const bitvector& mask) const;

    long selectValues(const bitvector& mask, void* vals) const;
    long selectValues(const bitvector& mask,
                      void* vals, array_t<uint32_t>& inds) const;

    virtual void write(FILE* file) const;
    virtual void print(std::ostream& out) const;
    void logMessage(const char* event, const char* fmt, ...) const;
    void logWarning(const char* event, const char* fmt, ...) const;

    int expandRange(ibis::qContinuousRange& rng) const;
    int contractRange(ibis::qContinuousRange& rng) const;

    virtual long estimateRange(const ibis::qContinuousRange& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;
    virtual long estimateRange(const ibis::qDiscreteRange& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;
    virtual long estimateRange(const ibis::qIntHod& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;
    virtual long estimateRange(const ibis::qUIntHod& cmp,
                               ibis::bitvector& low,
                               ibis::bitvector& high) const;

    virtual long evaluateRange(const ibis::qContinuousRange& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;
    virtual long evaluateRange(const ibis::qDiscreteRange& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;
    virtual long evaluateRange(const ibis::qIntHod& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;
    virtual long evaluateRange(const ibis::qUIntHod& cmp,
                               const ibis::bitvector& mask,
                               ibis::bitvector& res) const;

    virtual long stringSearch(const char*, ibis::bitvector&) const;
    virtual long stringSearch(const std::vector<std::string>&,
                              ibis::bitvector&) const;
    virtual long stringSearch(const char*) const;
    virtual long stringSearch(const std::vector<std::string>&) const;
    virtual long keywordSearch(const char*, ibis::bitvector&) const;
    virtual long keywordSearch(const char*) const;
    virtual long patternSearch(const char*) const;
    virtual long patternSearch(const char*, ibis::bitvector &) const;

    virtual long estimateRange(const ibis::qContinuousRange& cmp) const;
    virtual long estimateRange(const ibis::qDiscreteRange& cmp) const;
    virtual long estimateRange(const ibis::qIntHod& cmp) const;
    virtual long estimateRange(const ibis::qUIntHod& cmp) const;

    virtual double estimateCost(const ibis::qContinuousRange& cmp) const;
    virtual double estimateCost(const ibis::qDiscreteRange& cmp) const;
    virtual double estimateCost(const ibis::qIntHod& cmp) const;
    virtual double estimateCost(const ibis::qUIntHod& cmp) const;
    virtual double estimateCost(const ibis::qString&) const {
        return 0;}
    virtual double estimateCost(const ibis::qMultiString&) const {
        return 0;}

    virtual float getUndecidable(const ibis::qContinuousRange& cmp,
                                 ibis::bitvector& iffy) const;
    virtual float getUndecidable(const ibis::qDiscreteRange& cmp,
                                 ibis::bitvector& iffy) const;
    virtual float getUndecidable(const ibis::qIntHod& cmp,
                                 ibis::bitvector& iffy) const;
    virtual float getUndecidable(const ibis::qUIntHod& cmp,
                                 ibis::bitvector& iffy) const;

    virtual long append(const char* dt, const char* df, const uint32_t nold,
                        const uint32_t nnew, uint32_t nbuf, char* buf);

    virtual long append(const void* vals, const ibis::bitvector& msk);
    virtual long writeData(const char* dir, uint32_t nold, uint32_t nnew,
                           ibis::bitvector& mask, const void *va1,
                           void *va2=0);
    template <typename T>
    long castAndWrite(const array_t<double>& vals, ibis::bitvector& mask,
                      const T special);
    virtual long saveSelected(const ibis::bitvector& sel, const char *dest,
                              char *buf, uint32_t nbuf);
    virtual long truncateData(const char* dir, uint32_t nent,
                              ibis::bitvector& mask) const;

    virtual double getActualMin() const;
    virtual double getActualMax() const;
    virtual double getSum() const;
    long getCumulativeDistribution(std::vector<double>& bounds,
                                   std::vector<uint32_t>& counts) const;
    long getDistribution(std::vector<double>& bbs,
                         std::vector<uint32_t>& counts) const;
    class info;
    class indexLock;
    class mutexLock;

protected:
    // protected member variables
    const part* thePart; 
    ibis::bitvector mask_;
    ibis::TYPE_T m_type;
    std::string m_name; 
    std::string m_desc; 
    std::string m_bins; 
    bool m_sorted;      
    double lower;       
    double upper;       

    mutable ibis::index* idx;
    mutable ibis::util::sharedInt32 idxcnt;

    void logError(const char* event, const char* fmt, ...) const;
    long string2int(int fptr, dictionary& dic, uint32_t nbuf, char* buf,
                    array_t<uint32_t>& out) const;
    double computeMin() const;
    double computeMax() const;
    double computeSum() const;
    void actualMinMax(const char *fname, const ibis::bitvector& mask,
                      double &min, double &max) const;
    template <typename T>
    void actualMinMax(const array_t<T>& vals, const ibis::bitvector& mask,
                      double& min, double& max) const;
    template <typename T>
    T computeMin(const array_t<T>& vals,
                 const ibis::bitvector& mask) const;
    template <typename T>
    T computeMax(const array_t<T>& vals,
                 const ibis::bitvector& mask) const;
    template <typename T>
    double computeSum(const array_t<T>& vals,
                      const ibis::bitvector& mask) const;

    virtual int searchSorted(const ibis::qContinuousRange&,
                             ibis::bitvector&) const;
    virtual int searchSorted(const ibis::qDiscreteRange&,
                             ibis::bitvector&) const;
    virtual int searchSorted(const ibis::qIntHod&,
                             ibis::bitvector&) const;
    virtual int searchSorted(const ibis::qUIntHod&,
                             ibis::bitvector&) const;
    template <typename T> int
        searchSortedICC(const array_t<T>& vals,
                        const ibis::qContinuousRange& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedICD(const array_t<T>& vals,
                        const ibis::qDiscreteRange& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedICD(const array_t<T>& vals,
                        const ibis::qIntHod& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedICD(const array_t<T>& vals,
                        const ibis::qUIntHod& rng,
                        ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCC(const char* fname,
                         const ibis::qContinuousRange& rng,
                         ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCD(const char* fname,
                         const ibis::qDiscreteRange& rng,
                         ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCD(const char* fname,
                         const ibis::qIntHod& rng,
                         ibis::bitvector& hits) const;
    template <typename T> int
        searchSortedOOCD(const char* fname,
                         const ibis::qUIntHod& rng,
                         ibis::bitvector& hits) const;

    template <typename T> uint32_t
        findLower(int fdes, const uint32_t nr, const T tgt) const;
    template <typename T> uint32_t
        findUpper(int fdes, const uint32_t nr, const T tgt) const;
    template <typename T>
        long selectValuesT(const bitvector& mask, array_t<T>& vals) const;
    template <typename T>
        long selectValuesT(const bitvector& mask,
                           array_t<T>& vals, array_t<uint32_t>& inds) const;
    template <typename T>
        long selectToStrings(const bitvector& mask,
                             std::vector<std::string>& str) const;
    template <typename T>
        long appendValues(const array_t<T>&, const ibis::bitvector&);
    long appendStrings(const std::vector<std::string>&, const ibis::bitvector&);

    class readLock;
    class writeLock;
    class softWriteLock;
    friend class readLock;
    friend class writeLock;
    friend class indexLock;
    friend class mutexLock;
    friend class softWriteLock;

private:
    mutable pthread_rwlock_t rwlock;
    mutable pthread_mutex_t mutex;

    column& operator=(const column&); // no assignment
}; // ibis::column

class FASTBIT_CXX_DLLSPEC ibis::column::info {
 public:
    const char* name;           
    const char* description;    
    const double expectedMin;   
    const double expectedMax;   
    const ibis::TYPE_T type;    
    info(const ibis::column& col)
        : name(col.name()), description(col.description()),
          expectedMin(col.lowerBound()),
          expectedMax(col.upperBound()), type(col.type()) {};
    info(const info& rhs)
        : name(rhs.name), description(rhs.description),
        expectedMin(rhs.expectedMin),
        expectedMax(rhs.expectedMax),
        type(rhs.type) {};
 private:
    info();
    info& operator=(const info&);
}; // ibis::column::info

class ibis::column::indexLock {
public:
    ~indexLock();
    indexLock(const ibis::column* col, const char* m);
    const ibis::index* getIndex() const {return theColumn->idx;};

private:
    const ibis::column* theColumn;
    const char* mesg;

    indexLock();
    indexLock(const indexLock&);
    indexLock& operator=(const indexLock&);
}; // ibis::column::indexLock

class ibis::column::mutexLock {
public:
    mutexLock(const ibis::column* col, const char* m)
        : theColumn(col), mesg(m) {
        if (ibis::gVerbose > 9)
            col->logMessage("gainExclusiveAccess",
                            "pthread_mutex_lock for %s", m);
        int ierr = pthread_mutex_lock(&(col->mutex));
        if (0 != ierr)
            col->logWarning("gainExclusiveAccess", "pthread_mutex_lock for %s "
                            "returned %d (%s)", m, ierr, strerror(ierr));
    }
    ~mutexLock() {
        if (ibis::gVerbose > 9)
            theColumn->logMessage("releaseExclusiveAccess",
                                  "pthread_mutex_unlock for %s", mesg);
        int ierr = pthread_mutex_unlock(&(theColumn->mutex));
        if (0 != ierr)
            theColumn->logWarning("releaseExclusiveAccess",
                                  "pthread_mutex_unlock for %s returned %d "
                                  "(%s)", mesg, ierr, strerror(ierr));
    }

private:
    const ibis::column* theColumn;
    const char* mesg;

    mutexLock() {}; // no default constructor
    mutexLock(const mutexLock&) {}; // can not copy
    mutexLock& operator=(const mutexLock&);
}; // ibis::column::mutexLock

class ibis::column::writeLock {
public:
    writeLock(const ibis::column* col, const char* m);
    ~writeLock();

private:
    const ibis::column* theColumn;
    const char* mesg;

    writeLock();
    writeLock(const writeLock&);
    writeLock& operator=(const writeLock&);
}; // ibis::column::writeLock

class ibis::column::softWriteLock {
public:
    softWriteLock(const ibis::column* col, const char* m);
    ~softWriteLock();
    bool isLocked() const {return(locked==0);}

private:
    const ibis::column* theColumn;
    const char* mesg;
    const int locked;

    softWriteLock();
    softWriteLock(const softWriteLock&);
    softWriteLock& operator=(const softWriteLock&);
}; // ibis::column::softWriteLock

class ibis::column::readLock {
public:
    readLock(const ibis::column* col, const char* m);
    ~readLock();

private:
    const ibis::column* theColumn;
    const char* mesg;

    readLock();
    readLock(const readLock&);
    readLock& operator=(const readLock&);
}; // ibis::column::readLock

inline int ibis::column::elementSize() const {
    int sz = -1;
    switch (m_type) {
    case ibis::OID: sz = sizeof(rid_t); break;
    case ibis::INT: sz = sizeof(int32_t); break;
    case ibis::UINT: sz = sizeof(uint32_t); break;
    case ibis::LONG: sz = sizeof(int64_t); break;
    case ibis::ULONG: sz = sizeof(uint64_t); break;
    case ibis::FLOAT: sz = sizeof(float); break;
    case ibis::DOUBLE: sz = sizeof(double); break;
    case ibis::BYTE: sz = sizeof(char); break;
    case ibis::UBYTE: sz = sizeof(unsigned char); break;
    case ibis::SHORT: sz = sizeof(int16_t); break;
    case ibis::USHORT: sz = sizeof(uint16_t); break;
    case ibis::CATEGORY: sz = 0; break; // no fixed size per element
    case ibis::TEXT: sz = 0; break; // no fixed size per element
    default: sz = -1; break;
    }
    return sz;
} // ibis::column::elementSize

inline bool ibis::column::isFloat() const {
    return(m_type == ibis::FLOAT || m_type == ibis::DOUBLE);
} // ibis::column::isFloat

inline bool ibis::column::isInteger() const {
    return(m_type == ibis::BYTE || m_type == ibis::UBYTE ||
           m_type == ibis::SHORT || m_type == ibis::USHORT ||
           m_type == ibis::INT || m_type == ibis::UINT ||
           m_type == ibis::LONG || m_type == ibis::ULONG);
} // ibis::column::isInteger

inline bool ibis::column::isSignedInteger() const {
    return(m_type == ibis::BYTE || m_type == ibis::SHORT ||
           m_type == ibis::INT || m_type == ibis::LONG);
} // ibis::column::isSignedInteger

inline bool ibis::column::isUnsignedInteger() const {
    return(m_type == ibis::UBYTE || m_type == ibis::USHORT ||
           m_type == ibis::UINT || m_type == ibis::ULONG);
} // ibis::column::isUnsignedInteger

inline bool ibis::column::isNumeric() const {
    return(m_type == ibis::BYTE || m_type == ibis::UBYTE ||
           m_type == ibis::SHORT || m_type == ibis::USHORT ||
           m_type == ibis::INT || m_type == ibis::UINT ||
           m_type == ibis::LONG || m_type == ibis::ULONG ||
           m_type == ibis::FLOAT || m_type == ibis::DOUBLE);
} // ibis::column::isNumeric

// the operator to print a column to an output stream
inline std::ostream& operator<<(std::ostream& out, const ibis::column& prop) {
    prop.print(out);
    return out;
}

namespace ibis { // for template specialization
    template <> long column::selectToStrings<signed char>
    (const bitvector& mask, std::vector<std::string>& str) const;
    template <> long column::selectToStrings<unsigned char>
    (const bitvector& mask, std::vector<std::string>& str) const;
}
#endif // IBIS_COLUMN_H