vcf_ds.cpp

Go to the documentation of this file.

Go to the SVN repository for this file.

/*  $Id: vcf_ds.cpp 47479 2023-05-02 13:24:02Z ucko $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Authors:  Liangshou Wu
 *
 */
#include <ncbi_pch.hpp>
 
#include <gui/widgets/seq_graphic/vcf_ds.hpp>
#include <gui/widgets/seq_graphic/seqgraphic_job.hpp>
#include <gui/widgets/seq_graphic/vcf_glyph.hpp>
#include <gui/widgets/seq_graphic/vcf_histogram.hpp>
 
#include <gui/widgets/seq_graphic/graph_utils.hpp>
#include <gui/widgets/seq_graphic/seqgraphic_utils.hpp>
 
#include <gui/widgets/loaders/columnar_vcf_variants.hpp>
#include <gui/widgets/loaders/columnar_vcf_iterator.hpp>
 
#include <objmgr/util/feature.hpp>
#include <objmgr/util/sequence.hpp>
#include <objects/general/Object_id.hpp>
#include <objects/general/Dbtag.hpp>
#include <objects/seqalign/Std_seg.hpp>
#include <objects/seq/Annotdesc.hpp>
#include <objects/gbproj/AbstractProjectItem.hpp>
 
#include <gui/objutils/density_map.hpp>
 
#include <serial/serial.hpp>
#include <connect/ncbi_types.h>
#include <connect/ncbi_conn_stream.hpp>
#include <serial/objistr.hpp>
 
#include <gui/objutils/project_item_extra.hpp>
 
#include <chrono>
 
BEGIN_NCBI_SCOPE
USING_SCOPE(objects);
 
// the maximal number of objects allowed to show without smearing
static const int kMaxObjNum = 250000;
 
// the maximal number of children in parent/child situations dictated by approximation of screen size limits
static const size_t kMaxChildNum = 500;
// the zoom level that allows to show sequence
static const float kSeqZoomLevel = 0.125;
 
// create feature product mapping if scale (m_Window) is less than this value
static const float kMinScaleForMapping = 16.;
 
// the maximum number of features to show before switching to histogram mode
static const int kMaxFeatures = 500;
 
static CRef<CVcfVariant> s_CreateVcfGlyph(const objects::CSeq_id &seq_id, const CVCFVariant_CI &variant_CI)
{
    CRef<CVcfVariant> result{ new CVcfVariant( seq_id, variant_CI.GetVariantID(), variant_CI.GetPosition(), variant_CI.GetRef(), variant_CI.GetAlt()) };
    static const vector<string> info_fields{ "SVTYPE", "POS", "END", "CIPOS", "CIEND", "SVLEN", "CLNSIG", "CLNREVSTAT", "CLNVC", "CLNVCSO", "MC", "RS", "DBVARID" };
    if (variant_CI.IsSetInfo()) {
        auto &info_map = result->SetInfoColumns();
        for (const auto &field : info_fields) {
            if (!variant_CI.IsSetInfoField(field))
                continue;
            info_map[field] = variant_CI.GetInfoField(field);
        }
        result->ParseInfoColumns();
    }
    return result;
}
 
static void s_DoSearch(const CSeq_id* seq_id, const INonAsnTrackData::TDataBlob& vcf_blob, const string& sSearchTerm, list<CRef<CObject>>& results)
{
    results.clear();
    vector<unsigned> blob_pos;
    set<string> data_cols{ CVCFVariantsBase::sm_ID, CVCFVariantsBase::sm_REF, CVCFVariantsBase::sm_ALT };
    CVCFSlicedVariants variants(vcf_blob, nullptr, data_cols);
    variants.GetPositionsForVariant(sSearchTerm, blob_pos);
 
    for(auto pos: blob_pos) {
        TSeqRange variant_range(pos, pos);
        CVCFSlicedVariants range_variants(vcf_blob, &variant_range, data_cols, true);
        CVCFVariant_CI variant_CI(range_variants);
 
        while (variant_CI.IsValid()) {
            CRef<CVcfVariant> vcf_object{ s_CreateVcfGlyph(*seq_id, variant_CI) };
            if (vcf_object && vcf_object->GetID() == sSearchTerm) {
                results.push_back(vcf_object);
            }
            ++variant_CI;
        }
    }
}
 
 
const string CVcfTrackData::GetDataSourceTypeName() const
{
    return string(typeid(CVcfFeatureDSType).name());
}
 
// performs some sort of a search in data and place results into a list of objects
void CVcfTrackData::DoSearch(const CSeq_id* seq_id, const string& sSearchTerm, list<CRef<CObject>>& results) const
{
    s_DoSearch(seq_id, GetBlob(), sSearchTerm, results);
}
 
 
INonAsnTrackData* CVcfTrackDataFactory::CreateTrackData(const CSeq_id &seq_id, const CProjectItem &prj_item) const
{
    do {
        if (!prj_item.IsSetDescr()) {
//          LOG_POST(Info << "\tDeclined: No descr");
            break;
        }
        if (!prj_item.GetItem().IsOther()) {
//          LOG_POST(Info << "\tDeclined: Not other");
            break;
        }
 
        if (!NStr::StartsWith(prj_item.GetItem().GetOther().GetType(), "ColumnarVCF", NStr::eNocase)) {
//          LOG_POST(Info << "\tDeclined: Not ColumnarVCF: " << prj_item.GetItem().GetOther().GetType());
            break;
        }
 
        bool found{ false };
        for (const auto& annotdesc : prj_item.GetDescr()) {
            if (!annotdesc->IsSrc())
                continue;
            const auto& prj_seq_id = annotdesc->GetSrc();
            if (seq_id.AsFastaString() != prj_seq_id.AsFastaString()) {
//              LOG_POST(Info << "\tDeclined: seq_id no match: " << prj_seq_id.AsFastaString());
                continue;
            }
            found = true;
            break;
        }
        if (!found) {
//          LOG_POST(Info << "\tDeclined: seq_id mismatch");
            break;
        }
        string annot_name;
        if (!CProjectItemExtra::GetStr(prj_item, "AnnotName", annot_name)) {
//          LOG_POST(Info << "\tDeclined: cannot get prj_item annot_name");
            break;
        }
 
        string title;
        CProjectItemExtra::GetStr(prj_item, "OriginalFile", title);
 
//      LOG_POST(Info << "\tGetting track for: " << seq_id.AsFastaString() << ", title: " << title << ", annot_name: " << annot_name);
        
        return new CVcfTrackData{ title, annot_name, prj_item.GetItem().GetOther().GetData() };
 
    } while (false);
    
    return nullptr;
}
 
string CVcfTrackDataFactory::GetExtensionIdentifier() const
{
    static string sid("vcf_nonasn_track_data_factory_type");
    return sid;
}
 
string CVcfTrackDataFactory::GetExtensionLabel() const
{
    static string label("VCF Non-ASN1 Track Data Factory Type");
    return label;
}
 
///////////////////////////////////////////////////////////////////////////////
/// CVcfFeatureJob: a job class for loading annotated features from a given
/// sequence and visible range.
///////////////////////////////////////////////////////////////////////////////
class CVcfFeatureJob : public CSGAnnotJob
{
public:
    using TDataBlob = INonAsnTrackData::TDataBlob;
    CVcfFeatureJob(const string& desc, CBioseq_Handle handle,
                  const SAnnotSelector& sel, const TSeqRange& range,
                  TModelUnit window, int max_feat,
                  TJobToken token, const TDataBlob& vcf_blob, bool show_only_pathogenic);
 
 
protected:
    virtual EJobState x_Execute();
 
    
protected:
    TModelUnit  m_Window;       ///< current window for smear bars
    /// the maximum number of features allowed.
    /// -2: pack all features into one smear line
    /// -1: theoretically unlimited, practically there is one (kMaxObjNum)
    /// -0: pack all features into a histogram
    /// >0: any limit set from a user
    int         m_FeatLimit;
    const TDataBlob& m_VcfBlob;
    bool m_ShowOnlyPathogenic;
};
 
 
///////////////////////////////////////////////////////////////////////////////
/// CVcfFeatureDS
///////////////////////////////////////////////////////////////////////////////
CVcfFeatureDS::CVcfFeatureDS(CScope& scope, const CSeq_id& id)
    : CSGFeatureDS(scope, id)
{}
 
void CVcfFeatureDS::LoadFeatures(SAnnotSelector& sel,
                                const TSeqRange& range, TModelUnit window,
                                TJobToken token, int max_feat,
                                ELinkedFeatDisplay LinkedFeat,
                                const string&)
{
    if (sel.GetIncludedAnnotsNames().empty())
       return;
    const TDataBlob &blob = m_DataRegistry->GetNonAsnDataBlob(sel.GetIncludedAnnotsNames()[0].GetName());
    CRef<CVcfFeatureJob> job(
        new CVcfFeatureJob("VCF Feature", m_Handle, sel, range, window,
                          max_feat, token, blob, m_ShowOnlyPathogenic));
    job->SetGraphLevel(CGraphUtils::GetNearestLevel(m_GraphLevels, window));
    x_LaunchJob(*job);
}
 
void CVcfFeatureDS::SetNonAsnDataRegistry(const INonAsnDataRegistry *registry)
{
    m_DataRegistry.Reset(registry);
}
 
bool CVcfFeatureDS::HasNonAsnData(const string &annot_name) const
{
    _ASSERT(m_DataRegistry);
    return m_DataRegistry->HasNonAsnData(annot_name);
}
 
const INonAsnDataSource::TDataBlob& CVcfFeatureDS::GetNonAsnDataBlob(const string &annot_name) const
{
    _ASSERT(m_DataRegistry);
    return m_DataRegistry->GetNonAsnDataBlob(annot_name);
}
 
// performs some sort of a search in data and place results into a list of objects
void CVcfFeatureDS::DoSearch(const string &annot_name, const string& sSearchTerm, list<CRef<CObject>>& results) const
{
    results.clear();
    if(HasNonAsnData(annot_name)) {
        s_DoSearch(GetBioseqHandle().GetSeqId(), GetNonAsnDataBlob(annot_name), sSearchTerm, results);
    }
}
 
size_t CVcfFeatureDS::GetFeaturesCountInRange(const string &annot_name, const TSeqRange& range)
{
    _ASSERT(m_DataRegistry);
    const TDataBlob &blob = m_DataRegistry->GetNonAsnDataBlob(annot_name);
    CVCFSlicedVariants variants(blob);
    return variants.Count(TSeqRange(range.GetFrom()+1, range.GetTo()+1));
}
 
///////////////////////////////////////////////////////////////////////////////
/// CVcfFeatureDSType
///////////////////////////////////////////////////////////////////////////////
 
ISGDataSource*
CVcfFeatureDSType::CreateDS(SConstScopedObject& object) const
{
    const CSeq_id& id = dynamic_cast<const CSeq_id&>(object.object.GetObject());
//                cerr << "Creating DS for seq-id" << endl << 
//                      MSerial_AsnText << id << endl;
    return new CVcfFeatureDS(object.scope.GetObject(), id);
}
 
 
string CVcfFeatureDSType::GetExtensionIdentifier() const
{
    static string sid("vcf_feature_ds_type");
    return sid;
}
 
 
string CVcfFeatureDSType::GetExtensionLabel() const
{
    static string slabel("Graphical View VCF Feature Data Source Type");
    return slabel;
}
 
 
bool CVcfFeatureDSType::IsSharable() const
{
    return false;
}
 
 
CVcfFeatureJob::CVcfFeatureJob(const string& desc, CBioseq_Handle handle,
                             const SAnnotSelector& sel,
                             const TSeqRange& range, TModelUnit window,
                             int max_feat,
                             TJobToken token,
                             const TDataBlob& vcf_blob,
                             bool show_only_pathogenic)
    : CSGAnnotJob(desc, handle, sel, range)
    , m_Window(window)
    , m_FeatLimit(max_feat)
    , m_VcfBlob(vcf_blob)
    , m_ShowOnlyPathogenic(show_only_pathogenic)
{
    SetToken(token);
}
 
 
IAppJob::EJobState CVcfFeatureJob::x_Execute()
{
    CSGJobResult* result = new CSGJobResult();
    m_Result.Reset(result);
 
    SetTaskName("Loading VCF records...");
    SetTaskTotal(0);
    SetTaskCompleted(0);
 
    try {
        TSeqRange range(m_Range.GetFrom() + 1, m_Range.GetTo() + 1);
        set<string> data_cols{ CVCFVariantsBase::sm_ID, CVCFVariantsBase::sm_REF, CVCFVariantsBase::sm_ALT, CVCFVariantsBase::sm_INFO };
        // ERR_POST(Error << "CVcfFeatureJob::x_Execute(): loading m_VcfBlob.size(): " << m_VcfBlob.size());
 
        CVCFSlicedVariants variants(m_VcfBlob, &range, data_cols);
 
        int size = (int)variants.Count();
        if (size < 0) {
            LOG_POST(Info << "Negative size: " << size);
        }
        if (size > 0) {
            SetTaskTotal(size);
 
            // use the real feature limit
            if (m_FeatLimit > 0) {
                // we may want to adjust feature limit based on the
                // actual screen size (in pixels) for this case
                int new_limit = (int)(m_FeatLimit * m_Range.GetLength() / (1024 * m_Window));
                //ERR_POST(Error << "FL adjusted from " << m_FeatLimit << " to " << new_limit << ",w:" << m_Window << ",r:" << m_Range.GetLength());
                m_FeatLimit = max(m_FeatLimit, new_limit);
            }
            else if (m_FeatLimit == -1) {
                // unlimited, but still set a upper limit to guard against problem.
                m_FeatLimit = kMaxObjNum;
            }
 
            // show a histogram when there are too many features (VCF records), but balance this with
            // being zoomed in at a level when we want to show all records nevertheless (SV-4934)
            if (size > kMaxFeatures && m_Window > 1.0) {
                size_t BinWidth(round(m_Window));
 
                CHistogramGlyph::TMap dmap(m_Range.GetFrom(), m_Range.GetTo(), m_Window);
 
                auto CalculateHistogram = [&](CVcfHistogram::THistogramVector& vcf_histogram) {
                    auto opt_start = chrono::steady_clock::now();
                    variants.GetHistogram(vcf_histogram, static_cast<bm::bvector_size_type>(BinWidth));
                    auto diff_opt = chrono::steady_clock::now() - opt_start;
                    auto exec_time_ms = chrono::duration_cast<chrono::milliseconds>(diff_opt).count();
                    return exec_time_ms;
                };
 
                const auto &annot_name = m_Sel.GetIncludedAnnotsNames()[0].GetName();
                string cache_key = CVcfHistogram::GetCacheKey(annot_name, BinWidth);
                auto data = CGraphCache<CVcfHistogram>::GetInstance().GetCachedData(cache_key);
                if (data) {
                    data->GetHistogram(m_Range, dmap);
                }
                else {
                    CVcfHistogram::THistogramVector VCFHistogram;
                    auto hist_calc_time = CalculateHistogram(VCFHistogram);
                    CVcfHistogram::InitHistogramGlyph(BinWidth, m_Range, VCFHistogram, dmap);
                    // Cache the histogram of its calculation took more than 50 ms
                    if (hist_calc_time > 50) {
                        data = CGraphCache<CVcfHistogram>::GetInstance().GetData(cache_key);
                        _ASSERT(data);
                        data->SetData(VCFHistogram);
                        CGraphCache<CVcfHistogram>::GetInstance().SaveData(data);
                    }
                }
 
                CHistogramGlyph* hist_glyph = new CHistogramGlyph(dmap, CSeqFeatData::eSubtype_variation);
                CRef<CSeqGlyph> glyph(hist_glyph);
                if (glyph) {
                    result->m_ObjectList.push_back(glyph);
                }
            }
            else {
                CVCFVariant_CI variant_CI(variants);
                CRef<CSeq_id> seq_id(new CSeq_id);
                seq_id->Assign(*m_Handle.GetSeqId());
 
                while (variant_CI.IsValid()) {
                    CRef<CVcfVariant> var{ s_CreateVcfGlyph(*seq_id, variant_CI) };
                    // temporary protection against SV-4959, ignore such glyphs
                    if(var->GetPos() == size_t(-1)) {
                        LOG_POST(Error << "CVcfVariant with pos == -1: " << var->GetID());
                    }
                    else {
                        if (m_ShowOnlyPathogenic && var->IsClinVar()) {
                            const auto &info_fields = var->GetInfoColumns();
                            auto clnsig_it = info_fields.find("CLNSIG");
                            if (clnsig_it == info_fields.end()) {
                                ++variant_CI;
                                continue;
                            }
                            static set<string> pathogens = { "pathogenic/likely_pathogenic", "likely_pathogenic", "pathogenic" };
                            string clinsig = clnsig_it->second;
                            NStr::ToLower(clinsig);
                            if (0 == pathogens.count(clinsig)) {
                                ++variant_CI;
                                continue;
                            }
 
                        }
 
                        CRef<CSeqGlyph> glyph{ new CVcfGlyph(var.GetPointer()) };
                        if (glyph) {
                            result->m_ObjectList.push_back(glyph);
                        }
                    }
                    ++variant_CI;
                }
            }
        }
        SetTaskCompleted(size);
    } catch (CException& ex) {
        m_Error.Reset(new CAppJobError(ex.GetMsg()));
        return eFailed;
    } catch (std::exception& ex) {
        m_Error.Reset(new CAppJobError(ex.what()));
        return eFailed;
    }
    result->m_Token = m_Token;
    return eCompleted;
}
 
 
 
 
END_NCBI_SCOPE