seqdb_lmdb.cpp

Go to the documentation of this file.

Go to the SVN repository for this file.

/*  $Id: seqdb_lmdb.cpp 100751 2023-09-07 12:41:08Z boratyng $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Author: Christiam Camacho
 *
 */
 
/// @file seqdb_lmdb.cpp
/// Implements interface to interact with LMDB files
 
#include <ncbi_pch.hpp>
#include <objtools/blast/seqdb_reader/impl/seqdb_lmdb.hpp>
#include <objtools/blast/seqdb_reader/impl/seqdbgeneral.hpp>
#include <corelib/ncbifile.hpp>
#include <objects/seqloc/PDB_seq_id.hpp>
#include <cmath>
 
BEGIN_NCBI_SCOPE
 
#define SEQDB_LMDB_TIMING
#ifdef SEQDB_LMDB_TIMING
template<class T>
static string s_FormatNum(T value)
{
    CNcbiOstrstream oss;
    oss.imbue(std::locale(""));
    oss << std::fixed << value;
    return CNcbiOstrstreamToString(oss);
}
 
#define SPEED(time, nentries) s_FormatNum((size_t)((nentries)/(time)))
#endif /* SEQDB_LMDB_TIMING */
 
void CBlastLMDBManager::CBlastEnv::InitDbi(lmdb::env & env, ELMDBFileType file_type)
{
    auto txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
    if(file_type == eLMDB) {
        try {
        m_dbis[eDbiAcc2oid] = lmdb::dbi::open(txn, blastdb::acc2oid_str.c_str(), MDB_DUPSORT | MDB_DUPFIXED).handle();
        }
        catch (...){ /* It's ok not to have acc in a db */}
        m_dbis[eDbiVolname] = lmdb::dbi::open(txn, blastdb::volname_str.c_str(), MDB_INTEGERKEY).handle();
        m_dbis[eDbiVolinof] = lmdb::dbi::open(txn, blastdb::volinfo_str.c_str(), MDB_INTEGERKEY).handle();
    }
    else if (file_type == eTaxId2Offsets) {
        m_dbis[eDbiTaxid2offset] = lmdb::dbi::open(txn, blastdb::taxid2offset_str.c_str()).handle();
    }
    else {
        NCBI_THROW( CSeqDBException, eArgErr, "Invalid lmdb file type");
    }
    txn.commit();
    txn.reset();
    return;
}
 
CBlastLMDBManager::CBlastEnv::CBlastEnv(const string & fname, ELMDBFileType file_type, bool read_only, Uint8 map_size) :
        m_Filename(fname), m_FileType(file_type),m_Env(lmdb::env::create()), m_Count(1), m_ReadOnly(read_only)
{
    const MDB_dbi num_db(3);
    m_Env.set_max_dbs(num_db);
    m_dbis.resize(eDbiMax, UINT_MAX);
    if(m_ReadOnly) {
        CFile tf(fname);
        Uint8 readMapSize = (tf.GetLength()/10000 + 1) *10000;
        if (!tf.Exists()) {
            NCBI_THROW( CSeqDBException, eFileErr, "File " + fname + " not found. If you renamed any BLAST database files, please use original file names, and makeblastdb to rename the database. If you deleted any BLAST database files, you need to recreate the database.");
        }
        m_Env.set_mapsize(readMapSize);
        try {
            m_Env.open(m_Filename.c_str(), MDB_NOSUBDIR|MDB_NOLOCK|MDB_RDONLY, 0664);
        }
        catch (lmdb::error& e) {
            NCBI_THROW(CSeqDBException, eFileErr, "LMDB runtime error: " + (string)e.what());
        }
        InitDbi(m_Env,file_type);
    }
    else {
        LOG_POST(Info <<"Initial Map Size: " << map_size);
        /// map_size 0 means use lmdb default
        if(map_size != 0) {
            m_Env.set_mapsize(map_size);
        }
        try {
            m_Env.open(m_Filename.c_str(), MDB_NOSUBDIR , 0664);
        }
        catch (lmdb::error& e) {
            NCBI_THROW(CSeqDBException, eFileErr, "LMDB runtime error: " + (string)e.what());
        }
    }
}
 
CBlastLMDBManager::CBlastEnv::~CBlastEnv()
{
    for (unsigned int i=0; i < m_dbis.size(); i++){
        if (m_dbis[i] != UINT_MAX) {
            mdb_dbi_close(m_Env,m_dbis[i]);
        }
    }
    m_Env.close();
}
 
MDB_dbi CBlastLMDBManager::CBlastEnv::GetDbi(EDbiType dbi_type)
{
    if(m_dbis[dbi_type] == UINT_MAX) {
        string err = "DB contains no ";
        switch (dbi_type) {
        case eDbiVolinof:
        case eDbiVolname:
            err += "vol info.";
            break;
        case eDbiAcc2oid:
            err += "accession info.";
            break;
        case eDbiTaxid2offset:
            err += "tax id info";
            break;
        default:
            NCBI_THROW( CSeqDBException, eArgErr, "Invalid dbi type");
        }
        NCBI_THROW( CSeqDBException, eArgErr, err);
 
    }
    return m_dbis[dbi_type];
}
 
void CBlastLMDBManager::CBlastEnv::SetMapSize(Uint8 map_size)
{
    if(!m_ReadOnly) {
        m_Env.set_mapsize(map_size);
    }
}
 
CBlastLMDBManager & CBlastLMDBManager::GetInstance() {
    static CSafeStatic<CBlastLMDBManager> lmdb_manager;
    return lmdb_manager.Get();
}
 
lmdb::env & CBlastLMDBManager::GetReadEnvVol(const string & fname,  MDB_dbi & db_volname, MDB_dbi & db_volinfo)
{
    CBlastEnv* p = GetBlastEnv(fname, eLMDB);
    db_volinfo = p->GetDbi(CBlastEnv::eDbiVolinof);
    db_volname = p->GetDbi(CBlastEnv::eDbiVolname);
    return p->GetEnv();
}
lmdb::env & CBlastLMDBManager::GetReadEnvAcc(const string & fname, MDB_dbi & db_acc, bool* opened)
{
    CBlastEnv* p = GetBlastEnv(fname, eLMDB, opened);
    db_acc = p->GetDbi(CBlastEnv::eDbiAcc2oid);
    return p->GetEnv();
}
lmdb::env & CBlastLMDBManager::GetReadEnvTax(const string & fname, MDB_dbi & db_tax, bool* opened)
{
    CBlastEnv* p = GetBlastEnv(fname, eTaxId2Offsets, opened);
    db_tax = p->GetDbi(CBlastEnv::eDbiTaxid2offset);
    return p->GetEnv();
}
 
 
CBlastLMDBManager::CBlastEnv* CBlastLMDBManager::GetBlastEnv(const string & fname,
                                                             ELMDBFileType file_type,
                                                             bool* opened)
{
    CFastMutexGuard guard(m_Mutex);
    NON_CONST_ITERATE(list <CBlastEnv* >, itr, m_EnvList) {
        if((*itr)->GetFilename() == fname)  {
            (*itr)->AddReference();
            if ( opened && !*opened ) {
                (*itr)->AddReference();
                *opened = true;
            }
            return (*itr);
        }
    }
    CBlastEnv * p (new CBlastEnv(fname, file_type));
    m_EnvList.push_back(p);
    if ( opened && !*opened ) {
        p->AddReference();
        *opened = true;
    }
    return p;
}
 
lmdb::env & CBlastLMDBManager::GetWriteEnv(const string & fname, Uint8 map_size)
{
    CFastMutexGuard guard(m_Mutex);
    NON_CONST_ITERATE(list <CBlastEnv* >, itr, m_EnvList) {
        if((*itr)->GetFilename() == fname)  {
            (*itr)->AddReference();
            return (*itr)->GetEnv();
        }
    }
    CBlastEnv * p (new CBlastEnv(fname, eLMDBFileTypeEnd, false, map_size));
    m_EnvList.push_back(p);
    return p->GetEnv();
}
 
 
void CBlastLMDBManager::CloseEnv(const string & fname)
{
    CFastMutexGuard guard(m_Mutex);
    NON_CONST_ITERATE(list <CBlastEnv* >, itr, m_EnvList) {
        if((*itr)->GetFilename() == fname)  {
            if((*itr)->RemoveReference() == 0) {
                delete *itr;
                itr = m_EnvList.erase(itr);
                break;
            }
        }
    }
}
 
CBlastLMDBManager::~CBlastLMDBManager()
{
    NON_CONST_ITERATE(list <CBlastEnv* >, itr, m_EnvList) {
        delete *itr;
    }
    m_EnvList.clear();
}
 
CSeqDBLMDB::CSeqDBLMDB(const string & fname)
    : m_LMDBFile(fname),
      m_Oid2SeqIdsFile(GetFileNameFromExistingLMDBFile(fname, ELMDBFileType::eOid2SeqIds)),
      m_Oid2TaxIdsFile(GetFileNameFromExistingLMDBFile(fname, ELMDBFileType::eOid2TaxIds)),
      m_TaxId2OidsFile(GetFileNameFromExistingLMDBFile(fname, ELMDBFileType::eTaxId2Oids)),
      m_TaxId2OffsetsFile(GetFileNameFromExistingLMDBFile(fname, ELMDBFileType::eTaxId2Offsets)),
      m_LMDBFileOpened(false),
      m_NumOids(0)
{
}
 
CSeqDBLMDB::~CSeqDBLMDB()
{
    if ( m_LMDBFileOpened ) {
        CBlastLMDBManager::GetInstance().CloseEnv(m_LMDBFile);
        m_LMDBFileOpened = false;
    }
}
 
void 
CSeqDBLMDB::GetOid(const string & accession, vector<blastdb::TOid> & oids, const bool allow_dup) const
{
    try {
    oids.clear();
    {
    MDB_dbi dbi_handle;
    lmdb::env & env = CBlastLMDBManager::GetInstance().GetReadEnvAcc(m_LMDBFile, dbi_handle, &m_LMDBFileOpened);
    lmdb::dbi dbi(dbi_handle);
    auto txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
    auto cursor = lmdb::cursor::open(txn, dbi);
        
    string acc = accession;
    lmdb::val data2find(acc);
 
    if (cursor.get(data2find, MDB_SET)) {            
        lmdb::val k, val;          
        cursor.get(k, val, MDB_GET_CURRENT);
        const char* d = val.data();
        oids.push_back(((d[3] << 24)&0xFF000000) | ((d[2] << 16) & 0xFF0000) | ((d[1] << 8) & 0xFF00) | (d[0]&0xFF));
 
        if(allow_dup) {
            while (cursor.get(k,val, MDB_NEXT_DUP)) {
                d = val.data();
                oids.push_back(((d[3] << 24)&0xFF000000) | ((d[2] << 16) & 0xFF0000) | ((d[1] << 8) & 0xFF00) | (d[0]&0xFF));
            }
        }
    }
    cursor.close();
    txn.reset();
    }
    CBlastLMDBManager::GetInstance().CloseEnv(m_LMDBFile);
    } catch (lmdb::error & e) {
        string dbname;
        CSeqDB_Path(m_LMDBFile).FindBaseName().GetString(dbname);
        if(e.code() == MDB_NOTFOUND) {
            NCBI_THROW( CSeqDBException, eArgErr, "Seqid list specified but no accession table is found in " + dbname);
        }
        else {
            NCBI_THROW( CSeqDBException, eArgErr, "Accessions to Oids lookup error in " + dbname);
        }
    }
}
 
 
void CSeqDBLMDB::GetVolumesInfo(vector<string> & vol_names, vector<blastdb::TOid> & vol_num_oids)
{
    MDB_dbi db_volname_handle;
    MDB_dbi db_volinfo_handle;
    lmdb::env & env = CBlastLMDBManager::GetInstance().GetReadEnvVol(m_LMDBFile, db_volname_handle, db_volinfo_handle);
    vol_names.clear();
    vol_num_oids.clear();
    {
    auto txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
    lmdb::dbi db_volname(db_volname_handle);
    lmdb::dbi db_volinfo(db_volinfo_handle);
    MDB_stat volinfo_stat, volname_stat;
    lmdb::dbi_stat(txn, db_volinfo, &volinfo_stat);
    lmdb::dbi_stat(txn, db_volname, &volname_stat);
    if(volinfo_stat.ms_entries != volname_stat.ms_entries) {
        NCBI_THROW( CSeqDBException, eArgErr, "Volinfo error ");
    }
 
    vol_names.resize(volinfo_stat.ms_entries);
    vol_num_oids.resize(volinfo_stat.ms_entries);
 
    auto cursor_volname = lmdb::cursor::open(txn, db_volname);
    auto cursor_volinfo = lmdb::cursor::open(txn, db_volinfo);
    for (unsigned int i=0; i < volinfo_stat.ms_entries; i++) {
        lmdb::val data2find(&i, sizeof(Int4));
        if (cursor_volname.get(data2find, MDB_SET)) {
            {
                lmdb::val k, val;
                cursor_volname.get(k, val, MDB_GET_CURRENT);
                vol_names[i].assign(val.data(), val.size());
            }
            if (cursor_volinfo.get(data2find, MDB_SET)) {
                lmdb::val k, val;
                cursor_volinfo.get(k, val, MDB_GET_CURRENT);
                const char* d = val.data();
                vol_num_oids[i] = (((d[3] << 24)&0xFF000000) | ((d[2] << 16) & 0xFF0000) | ((d[1] << 8) & 0xFF00) | (d[0]&0xFF));
            }
            else {
                NCBI_THROW( CSeqDBException, eArgErr, "No volinfo for " + vol_names[i]);
            }
         }
    }
    cursor_volname.close();
    cursor_volinfo.close();
    txn.reset();
    }
    CBlastLMDBManager::GetInstance().CloseEnv(m_LMDBFile);
 
    for(unsigned int j=0; j < vol_num_oids.size(); j++){
        m_NumOids += vol_num_oids[j];
    }
}
 
void
CSeqDBLMDB::GetOids(const vector<string>& accessions, vector<blastdb::TOid>& oids) const
{
    try {
    oids.clear();
    oids.resize(accessions.size(), kSeqDBEntryNotFound);
 
    MDB_dbi dbi_handle;
    lmdb::env & env = CBlastLMDBManager::GetInstance().GetReadEnvAcc(m_LMDBFile, dbi_handle, &m_LMDBFileOpened);
    {
    lmdb::dbi dbi(dbi_handle);
    auto txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
 
    auto cursor = lmdb::cursor::open(txn, dbi);
 
    unsigned int i=0;
    for (i=0; i < accessions.size(); i++) {
        string acc = accessions[i];
        lmdb::val data2find(acc);
        if (cursor.get(data2find, MDB_SET)) {
            lmdb::val k, val;
            cursor.get(k, val, MDB_GET_CURRENT);
            const char* d = val.data();
            oids[i] = (((d[3] << 24)&0xFF000000) | ((d[2] << 16) & 0xFF0000) | ((d[1] << 8) & 0xFF00) | (d[0]&0xFF));
        }
    }
 
    cursor.close();
    txn.reset();
    }
    CBlastLMDBManager::GetInstance().CloseEnv(m_LMDBFile);
    } catch (lmdb::error & e) {
        string dbname;
        CSeqDB_Path(m_LMDBFile).FindBaseName().GetString(dbname);
        if(e.code() == MDB_NOTFOUND) {
            NCBI_THROW( CSeqDBException, eArgErr, "Seqid list specified but no accession table is found in " + dbname);
        }
        else {
            NCBI_THROW( CSeqDBException, eArgErr, "Accessions to Oids lookup error in " + dbname);
        }
    }
}
 
struct SOidSeqIdPair
{
    SOidSeqIdPair(blastdb::TOid o, const string & i) : oid(o), id(i) {}
    blastdb::TOid oid;
    string id;
    static bool cmp_oid(const SOidSeqIdPair & v, const SOidSeqIdPair & k) {
        if(v.oid == k.oid) {
            return (v.id < k.id);
        }
        return (v.oid < k.oid );
    }
};
 
class CLookupSeqIds
{
public:
    CLookupSeqIds(CMemoryFile & file): m_IndexStart((Uint8*) file.GetPtr()), m_DataStart((char *) file.GetPtr()) {
        if(m_IndexStart == NULL){
            NCBI_THROW( CSeqDBException, eArgErr, "Failed to open oid-to-seqid lookup file");
        }
 
        Uint8 num_of_oids = *m_IndexStart;
        m_IndexStart ++;
        m_DataStart += (8 * (num_of_oids + 1));
    }
 
    inline void GetSeqIdListForOid(blastdb::TOid oid, vector<string> & idlist);
private:
 
    Uint8 * m_IndexStart;
    char * m_DataStart;
};
 
void CLookupSeqIds::GetSeqIdListForOid(blastdb::TOid oid, vector<string> & idlist)
{
    Uint8 * index_ptr = m_IndexStart + oid;
    Char * end = m_DataStart + (*index_ptr);
    index_ptr--;
    Char * begin = (oid == 0) ? m_DataStart:m_DataStart + (*index_ptr);
    while (begin < end) {
        unsigned char id_len = *begin;
        begin ++;
        if(id_len == 0xFF) {
            Uint4 long_id_len = *((Uint4 *) begin);
            begin +=4;
            string id;
            id.assign(begin, long_id_len);
            begin += long_id_len;
            idlist.push_back(id);
        }
        else {
            string id;
            id.assign(begin, id_len);
            begin += id_len;
            idlist.push_back(id);
        }
    }
}
 
bool s_CompareIdList(vector<string> & file_idlist, vector<string> &input_idlist)
{
    bool rv = false;
    vector<string>::iterator f_itr = file_idlist.begin();
    vector<string>::iterator i_itr = input_idlist.begin();
    while(f_itr != file_idlist.end() && i_itr != input_idlist.end()) {
        if(*i_itr == *f_itr) {
            i_itr++;
            f_itr++;
            continue;
        }
        else {
            CSeq_id seq_id(*i_itr, CSeq_id::fParse_RawText | CSeq_id::fParse_AnyLocal | CSeq_id::fParse_PartialOK);
            // Input id is PDB with just mol id
            if(seq_id.IsPdb() &&  !seq_id.GetPdb().IsSetChain_id()) {
                CSeq_id file_seqid(*f_itr, CSeq_id::fParse_RawText | CSeq_id::fParse_AnyLocal | CSeq_id::fParse_PartialOK);
                if (file_seqid.IsPdb() && file_seqid.GetPdb().GetMol().Get() == *i_itr) {
                    f_itr++;
                    string tmp_pdb = *i_itr;
                    while ((f_itr != file_idlist.end()) && ((*f_itr).find_first_of(tmp_pdb) == 0)){
                        f_itr++;
                    }
                    // Skip pdb id in input list but with chain id
                    while ((i_itr != input_idlist.end()) && ((*i_itr).find_first_of(tmp_pdb) == 0)){
                        i_itr++;
                    }
                    continue;
                }
            }
            else {
                CSeq_id file_seq_id(*f_itr, CSeq_id::fParse_RawText | CSeq_id::fParse_AnyLocal | CSeq_id::fParse_PartialOK);
                if( file_seq_id.GetSeqIdString(false) == *i_itr) {
                    i_itr++;
                    // Skip identical id in input list but with version
                    if((i_itr != input_idlist.end()) && (file_seq_id.GetSeqIdString(true) == *i_itr)){
                        i_itr++;
                    }
                    f_itr++;
                    continue;
                }
            }
            break;
        }
    }
    if(f_itr == file_idlist.end()){
        rv=true;
    }
 
    file_idlist.clear();
    input_idlist.clear();
    return rv;
}
 
void
CSeqDBLMDB::NegativeSeqIdsToOids(const vector<string>& ids, vector<blastdb::TOid>& rv) const
{
    rv.clear();
    vector<blastdb::TOid> oids;
    GetOids(ids, oids);
    vector<SOidSeqIdPair> pairs;
    for (unsigned int i=0; i < ids.size(); i++) {
        if(oids[i] == kSeqDBEntryNotFound) {
            continue;
        }
        else {
            SOidSeqIdPair p(oids[i], ids[i]);
            pairs.push_back(p);
        }
    }
 
    if(pairs.size() == 0) {
        return;
    }
 
    sort (pairs.begin(), pairs.end(), SOidSeqIdPair::cmp_oid);
 
    CMemoryFile oid_file(m_Oid2SeqIdsFile);
    CLookupSeqIds lookup(oid_file);
    blastdb::TOid current_oid = 0;
    unsigned int i=0;
    while (i < pairs.size()) {
        vector<string> file_idlist;
        vector<string> input_idlist;
        current_oid = pairs[i].oid;
        lookup.GetSeqIdListForOid(current_oid, file_idlist);
        while ((i < pairs.size()) && (current_oid == pairs[i].oid)) {
            input_idlist.push_back(pairs[i].id);
            i++;
        }
        if(s_CompareIdList(file_idlist, input_idlist)) {
            rv.push_back(current_oid);
        }
 
    }
 
}
 
void CSeqDBLMDB::GetDBTaxIds(vector<TTaxId> & tax_ids) const
{
 
    tax_ids.clear();
    try {
    MDB_dbi dbi_handle;
    lmdb::env & env = CBlastLMDBManager::GetInstance().GetReadEnvTax(m_TaxId2OffsetsFile, dbi_handle);
    {
        auto txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
        auto dbi(dbi_handle);
        auto cursor = lmdb::cursor::open(txn, dbi);
        lmdb::val key;
        while (cursor.get(key, MDB_NEXT)) {
            TTaxId taxid = TAX_ID_FROM(Int4, *((Int4 *)key.data()));
            tax_ids.push_back(taxid);
        }
        cursor.close();
        txn.reset();
    }
    }
    catch (lmdb::error & e) {
        string dbname;
        CSeqDB_Path(m_TaxId2OffsetsFile).FindBaseName().GetString(dbname);
        if(e.code() == MDB_NOTFOUND) {
            NCBI_THROW( CSeqDBException, eArgErr, "No taxonomy info found in " + dbname);
        }
        else {
            NCBI_THROW( CSeqDBException, eArgErr, "Taxonomy Id to Oids lookup error in " + dbname);
        }
    }
    CBlastLMDBManager::GetInstance().CloseEnv(m_TaxId2OffsetsFile);
}
 
void CSeqDBLMDB::GetOidsForTaxIds(const set<TTaxId> & tax_ids, vector<blastdb::TOid>& oids, vector<TTaxId> & tax_ids_found) const
{
    try {
    oids.clear();
    tax_ids_found.clear();
    vector<Uint8> offsets;
    MDB_dbi dbi_handle;
    lmdb::env & env = CBlastLMDBManager::GetInstance().GetReadEnvTax(m_TaxId2OffsetsFile, dbi_handle);
    {
    auto txn = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
    lmdb::dbi dbi(dbi_handle);
    auto cursor = lmdb::cursor::open(txn, dbi);
    ITERATE(set<TTaxId>, itr, tax_ids) {
        Int4 tax_id = TAX_ID_TO(Int4, *itr);
        lmdb::val data2find(tax_id);
 
        if (cursor.get(data2find, MDB_SET)) {
            lmdb::val k, val;
            cursor.get(k, val, MDB_GET_CURRENT);
            const char* d = val.data();
            offsets.push_back((((Uint8) d[7] << 56) &0xFF00000000000000) | (((Uint8) d[6] << 48) & 0xFF000000000000) |
                              (((Uint8) d[5] << 40) &0xFF0000000000)     | (((Uint8) d[4] << 32) & 0xFF00000000)     |
                              (((Uint8) d[3] << 24) &0xFF000000)         | (((Uint8) d[2] << 16) & 0xFF0000)         |
                              (((Uint8) d[1] << 8)  &0xFF00)             | ((Uint8)  d[0]&0xFF));
            while (cursor.get(k,val, MDB_NEXT_DUP)) {
                d = val.data();
                offsets.push_back((((Uint8) d[7] << 56) &0xFF00000000000000) | (((Uint8) d[6] << 48) & 0xFF000000000000) |
                                  (((Uint8) d[5] << 40) &0xFF0000000000)     | (((Uint8) d[4] << 32) & 0xFF00000000)     |
                                  (((Uint8) d[3] << 24) &0xFF000000)         | (((Uint8) d[2] << 16) & 0xFF0000)         |
                                  (((Uint8) d[1] << 8)  &0xFF00)             | ((Uint8)  d[0]&0xFF));
            }
            tax_ids_found.push_back(*itr);
        }
    }
    cursor.close();
    txn.reset();
    }
    CBlastLMDBManager::GetInstance().CloseEnv(m_TaxId2OffsetsFile);
    vector<bool> oids_set(m_NumOids, false);
 
    CMemoryFile oid_file(m_TaxId2OidsFile);
    const char * start_ptr = (char *) oid_file.GetPtr();
    for (unsigned int i=0; i < offsets.size(); i++) {
        Uint4 * list_ptr = (Uint4 * ) (start_ptr + offsets[i]);
        Uint4 num_of_oids = *list_ptr;
        Uint4 count = 0 ;
        list_ptr ++;
        while(count < num_of_oids) {
            if(!oids_set[*list_ptr]) {
                oids.push_back(*list_ptr);
                oids_set[*list_ptr] = true;
            }
            count++;
            list_ptr++;
        }
    }
 
    int oids_sz = oids.size();
    if((oids_sz*log(oids_sz)) < m_NumOids) {
        sort(oids.begin(), oids.end());
    }
    else {
        oids.resize(0);
        oids.reserve(oids_sz);
        for (int i=0; i < oids_set.size(); i++) {
            if(oids_set[i]) {
                oids.push_back(i);
            }
        }
    }
 
    } catch (lmdb::error & e) {
        string dbname;
        CSeqDB_Path(m_TaxId2OffsetsFile).FindBaseName().GetString(dbname);
        if(e.code() == MDB_NOTFOUND) {
            NCBI_THROW( CSeqDBException, eArgErr, "No taxonomy info found in " + dbname);
        }
        else {
            NCBI_THROW( CSeqDBException, eArgErr, "Taxonomy Id to Oids lookup error in " + dbname);
        }
    }
}
 
 
class CLookupTaxIds
{
public:
    CLookupTaxIds(CMemoryFile & file): m_IndexStart((Uint8*) file.GetPtr()), m_DataStart((Int4 *) file.GetPtr()) {
        if(m_IndexStart == NULL){
            NCBI_THROW( CSeqDBException, eArgErr, "Failed to open oid-to-taxids lookup file");
        }
 
        Uint8 num_of_oids = *m_IndexStart;
        m_IndexStart ++;
        m_DataStart += (2* (num_of_oids + 1));
    }
 
    inline void GetTaxIdListForOid(blastdb::TOid oid, vector<TTaxId> & taxid_list);
private:
 
    Uint8 * m_IndexStart;
    Int4 * m_DataStart;
};
 
void CLookupTaxIds::GetTaxIdListForOid(blastdb::TOid oid, vector<TTaxId> & taxid_list)
{
    taxid_list.clear();
    Uint8 * index_ptr = m_IndexStart + oid;
    Int4 * end = m_DataStart + (*index_ptr);
    index_ptr--;
    Int4 * begin = (oid == 0) ? m_DataStart:m_DataStart + (*index_ptr);
    while (begin < end) {
        taxid_list.push_back(TAX_ID_FROM(Int4, *begin));
        begin++;
    }
}
 
void
CSeqDBLMDB::NegativeTaxIdsToOids(const set<TTaxId>& tax_ids, vector<blastdb::TOid>& rv, vector<TTaxId> & tax_ids_found) const
{
    rv.clear();
    vector<blastdb::TOid> oids;
    GetOidsForTaxIds(tax_ids, oids, tax_ids_found);
    CMemoryFile oid_file(m_Oid2TaxIdsFile);
    set<TTaxId> tax_id_list(tax_ids_found.begin(), tax_ids_found.end());
    CLookupTaxIds lookup(oid_file);
    for(unsigned int i=0; i < oids.size(); i++) {
        vector<TTaxId>  file_list;
        lookup.GetTaxIdListForOid(oids[i], file_list);
        if(file_list.size() > tax_ids.size()) {
            continue;
        }
        else {
            unsigned int j = 0;
            for(; j < file_list.size(); j++) {
                if(tax_id_list.find(file_list[j]) == tax_id_list.end()) {
                    break;
                }
            }
            if(j == file_list.size()) {
                rv.push_back(oids[i]);
            }
        }
    }
}
 
void CSeqDBLMDB::GetTaxIdsForOids(const vector<blastdb::TOid> & oids, set<TTaxId> & tax_ids) const
{
    CMemoryFile oid_file(m_Oid2TaxIdsFile);
    CLookupTaxIds lookup(oid_file);
    for(unsigned int i=0; i < oids.size(); i++) {
        vector<TTaxId>  taxid_list;
        lookup.GetTaxIdListForOid(oids[i], taxid_list);
        tax_ids.insert(taxid_list.begin(), taxid_list.end());
    }
}
 
 
string BuildLMDBFileName(const string& basename, bool is_protein, bool use_index, unsigned int index)
{
    if (basename.empty()) {
        throw invalid_argument("Basename is empty");
    }
 
    string vol_str=kEmptyStr;
    if(use_index) {
        vol_str = (index > 9) ?".": ".0";
        vol_str += NStr::UIntToString(index);
    }
    return basename + vol_str + (is_protein ? ".pdb" : ".ndb");
}
 
string GetFileNameFromExistingLMDBFile(const string& lmdb_filename, ELMDBFileType file_type)
{
 
    string filename (lmdb_filename, 0, lmdb_filename.size() - 2);
    switch (file_type) {
        case eLMDB :
            filename += "db";
        break;
        case eOid2SeqIds :
            filename +="os";
        break;
        case eOid2TaxIds :
            filename +="ot";
        break;
        case eTaxId2Offsets :
            filename += "tf";
        break;
        case eTaxId2Oids :
            filename += "to";
        break;
        default :
            NCBI_THROW( CSeqDBException, eArgErr, "Invalid LMDB file type");
        break;
    }
    return filename;
}
 
void DeleteLMDBFiles(bool db_is_protein, const string & filename)
{
    vector<string> extn;
    SeqDB_GetLMDBFileExtensions(db_is_protein, extn);
    ITERATE(vector<string>, itr, extn) {
         CFile f(filename + "." + (*itr));
         if (f.Exists()) {
             f.Remove();
         }
    }
}
 
 
END_NCBI_SCOPE