columnar_vcf_variants.cpp

Go to the documentation of this file.

Go to the SVN repository for this file.

/*  $Id: columnar_vcf_variants.cpp 47479 2023-05-02 13:24:02Z ucko $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Authors:  Andrea Asztalos, Anatoliy Kuznetsov
 *
 * File Description:
 *   
 *
 */
 
#include <ncbi_pch.hpp>
#include <util/line_reader.hpp>
#include <util/bitset/ncbi_bitset.hpp>
#include <util/bitset/bmdbg.h>
#include <gui/widgets/loaders/columnar_vcf_ex.hpp>
#include <gui/widgets/loaders/columnar_vcf_iterator.hpp>
#include <gui/widgets/loaders/columnar_vcf_variants.hpp>
#include <chrono>
#include <numeric>
 
BEGIN_NCBI_SCOPE
 
CVCFVariantList::CVCFVariantList(const string& chr_name, 
    bool load_all_info, 
    const set<string>& info_fields,
    const map<unsigned, string>& sample_cols)
    : CVCFVariantsBase(load_all_info, info_fields, sample_cols), m_ChrName(chr_name)
{
}
 
CVCFVariantList::CVCFVariantList(const string& chr_name, const string& filename)
    : m_ChrName(chr_name)
{
    CFileIO fio;
    try {
        fio.Open(filename, CFileIO::eOpen, CFileIO::eRead);
    }
    catch (const CException& e) {
        NCBI_THROW(CVCFDataException, eFileCorrupted, "Cannot open " + filename + "\nas: " + e.GetMsg());
    }
    size_t total_size = fio.GetFileSize();
 
    char* buf_ptr = new char[total_size];
    try {
        fio.Read(buf_ptr, total_size);
    }
    catch (const CException& e) {
        NCBI_THROW(CVCFDataException, eFileCorrupted, e.GetMsg());
    }
 
    m_Buffer = new vector<char>(buf_ptr, buf_ptr + total_size);
    _ASSERT(m_Buffer->size() == total_size);
 
    delete[] buf_ptr;
    buf_ptr = nullptr;
 
    fio.Close();
    x_DeserializeAllData();
    m_Index = 0;
}
 
CVCFVariantList::CVCFVariantList(const string& chr_name, const vector<char>& data)
    : m_ChrName(chr_name)
{
    m_Buffer = new vector<char>(data.begin(), data.end());
    x_DeserializeAllData();
    m_Index = 0;
}
 
CVCFVariantList::CVCFVariantList(const vector<char>& data, const string& version)
{
    if (version != s_GetPreviousVersion()) {
        NCBI_THROW(CColumnarVCFException, eConversionFailed, "Trying to open a project with version: " + version);
    }
 
    const unsigned char* buf_ptr = (const unsigned char*)data.data();
    size_t nr_cols = 0;
    bool skip_max_feat_length = true;
    x_DeserializeIndexVectors(buf_ptr, nr_cols, skip_max_feat_length);
 
    x_DeserializeColumn(CVCFVariantsBase::sm_ID, buf_ptr, nr_cols);
    auto var_it = m_Descriptors[CVCFVariantsBase::sm_ID].begin();
    m_StartPos = m_Posindexmap.GetPositionForIndex(var_it.pos());
    m_StopPos = m_Posindexmap.GetPositionForIndex(m_Posindexmap.GetMaxIndex());
    _ASSERT(m_StartPos <= m_StopPos);
 
    m_MaxFeatLength = 0;
    bool found_end_col = x_DeserializeColumn("END", buf_ptr, nr_cols);
    if (!found_end_col) {
        // when the 'END' Info subfield is not present, the data can be converted with m_MaxFeatLength = 0
    }
    else {
        // When 'END' subfield is present, we need to fetch the start positions of the variants to 
        // calculate m_MaxFeatLength. This is time expensive due to the way these positions are stored.
        // Therefore we will convert only those VCF blobs that don't contain more than a given number of variants
        if (Count() > 100000) {
            NCBI_THROW(CColumnarVCFException, eConversionFailed,
                "Too many VCF variants to be converted: " + NStr::UIntToString(Count()));
        }
        else {
         // calculate the max feat length
            auto start = chrono::steady_clock::now();
            const auto& end_vector = m_Descriptors.GetInfoField("END");
 
            auto end_it = end_vector.begin();
            auto end_stop_it = end_vector.end();
            m_MaxFeatLength = 0;
 
            for (; end_it != end_stop_it; ++end_it) {
                if (!end_it.is_null()) {
                    try {
                        unsigned end_point = NStr::StringToUInt(*end_it);
                        unsigned start_point = m_Posindexmap.GetPositionForIndex(end_it.pos());
                        int tmp_delta = end_point - start_point + 1;
                        if (tmp_delta < 0) {
                            tmp_delta = tmp_delta * (-1);
                        }
                        m_MaxFeatLength = max((unsigned)tmp_delta, m_MaxFeatLength);
                    }
                    catch (const CException&) {}
                }
            }
            auto diff = chrono::steady_clock::now() - start;
            LOG_POST(Info << "Finding maxfeat length for " << Count() << " variants took: "
                << chrono::duration_cast<chrono::milliseconds>(diff).count() << "ms");
        }
    }
 
    // store the serialized data in the current format
    size_t total_size = data.size() + sizeof(unsigned); // for additionally saving m_MaxFeatLength
   
    m_Buffer = new vector<char>(total_size);
    char* write_ptr = m_Buffer->data();
    buf_ptr = (const unsigned char*)data.data();
    
    // save the number of index vectors
    size_t idx_size = GetNumberOfIndexVecs();
    memcpy(write_ptr, &idx_size, sizeof(idx_size));
    write_ptr += sizeof(idx_size);
    buf_ptr += sizeof(idx_size);
 
    // save the number of string columns
    memcpy(write_ptr, &nr_cols, sizeof(nr_cols));
    write_ptr += sizeof(nr_cols);
    buf_ptr += sizeof(nr_cols);
 
    // save the maximum feature length
    memcpy(write_ptr, &m_MaxFeatLength, sizeof(m_MaxFeatLength));
    write_ptr += sizeof(m_MaxFeatLength);
 
    memcpy(write_ptr, buf_ptr, data.size() - sizeof(idx_size) - sizeof(nr_cols));
}
 
CVCFVariantList::~CVCFVariantList()
{
    // delete the actual data as well
    if (m_Buffer) {
        delete m_Buffer;
        m_Buffer = nullptr;
    }
}
 
void CVCFVariantList::ParseLine(const string& line)
{
    size_t pos = line.find("\t");
    _ASSERT(m_ChrName == line.substr(0, pos));
 
    if (!m_Descriptors.AreInsertersReady()) {
        m_Descriptors.InitInserters();
    }
 
    string remainder = line;
    // parsing the POS column
    string value = x_ParseNextColumn(remainder, pos);
    m_StopPos = NStr::StringToUInt(value, NStr::fConvErr_NoThrow);
    bool added = (errno == 0);
    if (added) {
        added = m_Posindexmap.Add(m_StopPos, static_cast<unsigned int>(m_Index));
        m_Descriptors.PushBackPos(m_StopPos);
    }
    if (!added) {
        NCBI_THROW(CColumnarVCFException, eParsePosFailed, "Failed to parse POS value:\n" + line);
    }
 
    // parsing the ID column
    value = x_ParseNextColumn(remainder, pos);
    vector<string> var_ids;
    if (value.find(";") == NPOS) { // there is a single variant ID
        var_ids.push_back(value);
    }
    else {
        NStr::Split(value, ";", var_ids);
        for (auto& it : var_ids) {
            NStr::TruncateSpacesInPlace(it);
        }
        sort(var_ids.begin(), var_ids.end());
        var_ids.erase(unique(var_ids.begin(), var_ids.end()), var_ids.end());
    }
 
    _ASSERT(!var_ids.empty());
 
    const auto& col_names = CVCFVariantsBase::s_GetAllColNames();
    auto col_it = col_names.begin();
    m_Descriptors.PushBack(*col_it, var_ids[0]);
    m_Descriptors.SetIndex() = m_Index;
 
    if (var_ids.size() == 1) {
        for (++col_it; col_it != col_names.end() - 1; ++col_it) {
            value = x_ParseNextColumn(remainder, pos);
            m_Descriptors.PushBack(*col_it, value);
        }
        if (pos == NPOS) {
            value = kEmptyStr;
        }
        else {
            value = remainder.substr(pos + 1, NPOS);
        }
        m_Descriptors.PushBack(*col_it, value);
    }
    else {
        vector<string> tmp_values;
        for (++col_it; col_it != col_names.end() - 1; ++col_it) {
            value = x_ParseNextColumn(remainder, pos);
            m_Descriptors.PushBack(*col_it, value);
            tmp_values.push_back(value);
        }
        if (pos == NPOS) {
            value = kEmptyStr;
        }
        else {
            value = remainder.substr(pos + 1, NPOS);
        }
        m_Descriptors.PushBack(*col_it, value);
        tmp_values.push_back(value);
 
        for (size_t index = 1; index < var_ids.size(); ++index) {
            m_Index++;
            m_Posindexmap.Add(m_StopPos, static_cast<unsigned int>(m_Index));
            m_Descriptors.PushBackPos(m_StopPos);
 
            col_it = col_names.begin();
            m_Descriptors.PushBack(*col_it, var_ids[index]);
 
            size_t n = 0;
            for (++col_it; col_it != col_names.end() && n < tmp_values.size(); ++col_it, ++n) {
                value = tmp_values[n];
                m_Descriptors.PushBack(*col_it, value);
            }
        }
    }
 
    // prepare the index for the next line
    ++m_Index;
}
 
void CVCFVariantList::FinalizeReading()
{
    m_Descriptors.FinalizeReading();
    m_Posindexmap.FinalizeReading();
    m_MaxFeatLength = m_Descriptors.GetMaxFeatureLength();
 
    m_Index = 0;
    auto var_it = m_Descriptors[CVCFVariantsBase::sm_ID].begin();
    m_StartPos = m_Posindexmap.GetPositionForIndex(var_it.pos());
    _ASSERT(m_StartPos <= m_StopPos);
}
 
string CVCFVariantList::x_ParseNextColumn(string& line, size_t& pos)
{
    if (pos == NPOS) {
        line.resize(0);
        return CVCFVariantsBase::sm_MissingValue;
    }
 
    line = line.substr(pos + 1, NPOS);
    pos = line.find("\t");
    /*
    if (pos == NPOS) {
        NCBI_THROW(CColumnarVCFException, eParseTabExpected, "Line expected to be tab delimited");
    }
    */
    return line.substr(0, pos);
}
 
bool CVCFVariantList::AreVariantIdsUnique(vector<pair<string, unsigned>>& copies)
{
    set<string> tmp;
    copies.clear();
 
    auto it = m_Descriptors[CVCFVariantsBase::sm_ID].begin();
    auto it_end = m_Descriptors[CVCFVariantsBase::sm_ID].end();
    for (; it != it_end; ++it) {
        if (!it.is_null()) {
            auto ret = tmp.insert(*it);
            if (ret.second == false) {
                string search(*it);
                auto copies_it = find_if(copies.begin(), copies.end(),
                    [&search](const pair<string, unsigned>& elem) { return elem.first == search; });
                if (copies_it == copies.end()) {
                    copies.emplace_back(search, 2);
                }
                else {
                    copies_it->second++;
                }
            }
        }
    }
    return (copies.empty());
}
 
void CVCFVariantList::GetPositionsForMissingVarID(vector<unsigned>& positions)
{
    TSparseStrVector::bvector_type indices;
    m_Descriptors.GetIndicesForMissingVarID(indices);
    if (indices.count() > 0) {
        m_Posindexmap.Lookup(indices, positions);
    }
}
 
bool CVCFVariantList::GetPositionsForVariant(const string& variant_id, vector<unsigned>& positions)
{
    if (variant_id == CVCFVariantsBase::sm_MissingValue) {
        GetPositionsForMissingVarID(positions);
        return true;
    }
    
    TSparseStrVector::bvector_type indices;
    bool found = m_Descriptors.GetIndicesForVariant(variant_id, indices);
    if (found) {
        m_Posindexmap.Lookup(indices, positions);
    }
    sort(positions.begin(), positions.end());
    positions.erase(unique(positions.begin(), positions.end()), positions.end());
    return found;
}
 
const vector<char>& CVCFVariantList::GetSerializedData() const
{
    if (!m_Buffer || m_Buffer->empty()) {
        x_SerializeData();
    }
    return *m_Buffer;
}
 
void CVCFVariantList::WriteSerializedData(const string& filename)
{
    if (!m_Buffer || m_Buffer->empty()) {
        x_SerializeData();
    }
 
    _ASSERT(!m_Buffer->empty());
    NVcfUtil::PrintToFile(m_Buffer->data(), m_Buffer->size(), filename);
}
 
bool CVCFVariantList::operator==(const CVCFVariantList& other) const
{
    return (m_ChrName == other.m_ChrName) &&
        (m_Index == other.m_Index) &&
        (m_StartPos == other.m_StartPos) &&
        (m_StopPos == other.m_StopPos) &&
        (m_Descriptors == other.m_Descriptors) &&
        (m_Posindexmap == other.m_Posindexmap);
}
 
void CVCFVariantList::List(CNcbiOstream& out, bool only_sv_cols) const
{
    for (CVCFVariant_CI var_iter(*this); var_iter; ++var_iter) {
        out << var_iter.GetPosition() << "\t" << var_iter.GetVariantID() << "\t"
            << var_iter.GetRef() << "\t" << var_iter.GetAlt();
        if (!only_sv_cols) {
            out << "\t" << var_iter.GetQual() << "\t" << var_iter.GetFilter()
                << "\t" << var_iter.GetInfo() << "\t" << var_iter.GetFormat();
        } 
        out << endl;
    }
}
 
void CVCFVariantList::ListSamples(CNcbiOstream& out) const
{
    auto samples = m_Descriptors.GetSampleNames();
    for (const auto& it : samples) {
        out << it << "\t";
    }
    out << endl;
    for (CVCFVariant_CI var_iter(*this); var_iter; ++var_iter) {
        out << var_iter.GetPosition() << "\t" << var_iter.GetVariantID() << "\t";
        for (const auto& it : samples) {
            out << var_iter.GetSample(it) << "\t";
        }
        out << endl;
    }
}
 
vector<string> CVCFVariantList::GetAllVariantIDS() const
{
    vector<string> variants;
    for (CVCFVariant_CI var_iter(*this); var_iter; ++var_iter) {
        variants.push_back(var_iter.GetVariantID());
    }
    return variants;
}
 
 
void CVCFVariantList::GetStatistics(CNcbiOstream& out)
{
    bm::bv_statistics stat_sum;
    stat_sum.reset();
    out << "Resetting stat_sum...." << endl;
    NVcfUtil::PrintStats(stat_sum, out);
    bool Jaccard_index = false;
 
    const auto& col_names = CVCFVariantsBase::s_GetColNames();
    for (const auto& it : col_names) {
        auto& str_vector = m_Descriptors[it];
        out << "-----------------------" << it << " vector-------------------------" << endl;
        LOG_POST(Info << "-----------------------" << it << " vector-------------------------");
        bm::print_svector_stat(
#if defined(BM_SCALAR_VERSION)  &&  BM_SCALAR_VERSION >= 0x070903
// Slight anachronism -- if BM_SCALAR_VERSION *is* defined, it'll be
// at least 0x070a04.
            out,
#endif
            str_vector, Jaccard_index);
        NVcfUtil::AddStats(stat_sum, str_vector, out);
    }
    m_Posindexmap.GetStatistics(stat_sum, Jaccard_index, out);
}
 
bool CVCFVariantList::SerializeVariantData(const string& prefix, CNcbiOstream* out)
{
    // prefix of descriptor file names
    string label = x_GetFilePrefix(prefix);
    bool status = true;
 
    unsigned cum_memory_used = 0;
    unsigned cum_layout_size = 0;
 
    if (out) {
        *out << "\nStarting to serialize columns for chr: " << m_ChrName << endl;
    }
 
    try {
        m_Posindexmap.SerializeVectors(label, out, cum_memory_used, cum_layout_size);
 
        const auto& col_names = CVCFVariantsBase::s_GetColNames();
        for (const auto& it : col_names) {
            auto& str_vector = m_Descriptors[it];
            NVcfUtil::SerializeColumn(str_vector, label, it, out, cum_memory_used, cum_layout_size);
        }
 
        const auto info_fields = m_Descriptors.GetInfoFieldNames();
        for (const auto& it : info_fields) {
            auto& info_subcol = m_Descriptors.GetInfoField(it);
            NVcfUtil::SerializeColumn(info_subcol, label, it, out, cum_memory_used, cum_layout_size);
        }
 
        const auto samples = m_Descriptors.GetSampleNames();
        for (const auto& it : samples) {
            auto& sample_col = m_Descriptors.GetSample(it);
            NVcfUtil::SerializeColumn(sample_col, label, it, out, cum_memory_used, cum_layout_size);
        }
 
        if (out) {
            *out << endl << "Total memory used: " << cum_memory_used << endl << endl;
            *out << "Total layout size: " << cum_layout_size << endl;
        }
    }
    catch (const CException& e) {
        status = false;
        LOG_POST(Info << "Serialization of data has failed: " << e.GetMsg());
    } 
    catch (const exception& e) {
        status = false;
        LOG_POST(Info << "Serialization of data has failed: " << e.what());
    }
    return status;
}
 
bool CVCFVariantList::DeserializeAndCheck(const string& prefix, CNcbiOstream* out)
{
    string label = x_GetFilePrefix(prefix);
    bool status = true;
 
    if (out) {
        *out << "\nStarting to deserialize blobs for chr: " << m_ChrName << endl;
    }
 
    try {
        m_Posindexmap.DeserializeVectors(label, out);
 
        const auto& col_names = CVCFVariantsBase::s_GetColNames();
        for (const auto& it : col_names) {
            auto& str_vector = m_Descriptors[it];
            NVcfUtil::DeserializeColumn(str_vector, label, it, out);
        }
 
        const auto info_fields = m_Descriptors.GetInfoFieldNames();
        for (const auto& it : info_fields) {
            auto& info_subcol = m_Descriptors.SetInfoField(it);
            NVcfUtil::DeserializeColumn(info_subcol, label, it, out);
        }
        
        const auto samples = m_Descriptors.GetSampleNames();
        for (const auto& it : samples) {
            auto& sample_col = m_Descriptors.SetSample(it);
            NVcfUtil::DeserializeColumn(sample_col, label, it, out);
        }
    }
    catch (const CException& e) {
        status = false;
        LOG_POST(Info << "Data deserialization has failed: " << e.GetMsg());
    }
    catch (const exception& e) {
        status = false;
        LOG_POST(Info << "Data deserialization has failed: " << e.what());
    }
    return status;
}
 
bool CVCFVariantList::RemoveSerializedOutput(const string& prefix)
{
    string label = x_GetFilePrefix(prefix);
    bool status = true;
    try {
        const auto& col_names = CVCFVariantsBase::s_GetColNames();
        for (const auto& it : col_names) {
            status = status && NVcfUtil::RemoveFile(NVcfUtil::GenerateColFileName(label, it));
        }
 
        status = status && m_Posindexmap.RemoveSerializedOutput(label);
 
        const auto info_fields = m_Descriptors.GetInfoFieldNames();
        for (const auto& it : info_fields) {
            status = status && NVcfUtil::RemoveFile(NVcfUtil::GenerateColFileName(label, it));
        }
 
        const auto samples = m_Descriptors.GetSampleNames();
        for (const auto& it : samples) {
            status = status && NVcfUtil::RemoveFile(NVcfUtil::GenerateColFileName(label, it));
        }
    }
    catch (const CException& e) {
        status = false;
        LOG_POST(Info << "Could not delete file: " << e.GetMsg());
    }
    catch (const exception& e) {
        status = false;
        LOG_POST(Info << "Could not delete file: " << e.what());
    }
    return status;
}
 
string CVCFVariantList::x_GetFilePrefix(const string& prefix) const
{
    return prefix + "_" + m_ChrName;
}
 
 
 
// CVCFSlicedVariants
 
CVCFSlicedVariants::CVCFSlicedVariants(const vector<char>& data, const TSeqRange* range, const set<string>& cols_to_decode, bool only_start)
    : m_ColsDecode(cols_to_decode)
{
    m_BufferPtr = (const unsigned char*)data.data();
    x_DeserializeIndexVectors(m_BufferPtr, m_NrCols);
    x_DeserializeDescr_Range(m_BufferPtr, m_NrCols, range, m_ColsDecode, only_start);
}
 
bool CVCFSlicedVariants::GetPositionsForVariant(const string& variant_id, vector<unsigned>& positions)
{
    if (variant_id == CVCFVariantsBase::sm_MissingValue) {
        GetPositionsForMissingVarID(positions);
        return true;
    }
    
    TSparseStrVector::bvector_type indices;
    bool found = m_Descriptors.GetIndicesForVariant(variant_id, indices);
    if (!found) {
        // deserialize the full ID vector and search in it
        x_DeserializeColumn(sm_ID, m_BufferPtr, m_NrCols);
        found = m_Descriptors.GetIndicesForVariant(variant_id, indices);
    }
 
    if (found) {
        m_Posindexmap.Lookup(indices, positions);
    }
    sort(positions.begin(), positions.end());
    positions.erase(unique(positions.begin(), positions.end()), positions.end());
    return found;
}
 
void CVCFSlicedVariants::GetPositionsForMissingVarID(vector<unsigned>& positions)
{
    // deserialize the full ID vector and search in it
    x_DeserializeColumn(sm_ID, m_BufferPtr, m_NrCols);
    TSparseStrVector::bvector_type indices;
    m_Descriptors.GetIndicesForMissingVarID(indices);
    if (indices.count() > 0) {
        m_Posindexmap.Lookup(indices, positions);
    }
}
 
void CVCFSlicedVariants::List(CNcbiOstream& out) const
{
    for (CVCFVariant_CI var_iter(*this); var_iter; ++var_iter) {
        out << var_iter.GetPosition() << "\t" << var_iter.GetVariantID() << "\t"
            << var_iter.GetRef() << "\t" << var_iter.GetAlt() << "\t"
            << var_iter.GetQual() << "\t" << var_iter.GetFilter() << "\t"
            << var_iter.GetInfo();
        if (var_iter.IsSetFormat()) {
            out << "\t" << var_iter.GetFormat() << "\t" << var_iter.GetSampleCols();
        }
        out << endl;
    }
}
 
 
END_NCBI_SCOPE