demo_read_large_vcf.cpp

Go to the documentation of this file.

Go to the SVN repository for this file.

/*  $Id: demo_read_large_vcf.cpp 46618 2021-08-05 19:04:11Z asztalos $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Authors:  Andrea Asztalos
 *
 * File Description:
 *    Demo application for reading large (> 100MB) VCF files
 */
 
 
#include <ncbi_pch.hpp>
 
#include <corelib/ncbiapp.hpp>
#include <corelib/ncbireg.hpp>
#include <corelib/ncbi_limits.hpp>
#include <util/compress/stream_util.hpp>
#include <gui/widgets/wx/compressed_file.hpp>
#include <gui/widgets/loaders/report_loader_errors.hpp>
#include <gui/widgets/loaders/columnar_vcf_variants.hpp>
#include <gui/widgets/loaders/columnar_vcf_reader.hpp>
#include <chrono>
 
USING_NCBI_SCOPE;
 
class CColumnarVCFReaderApp : public CNcbiApplication
{
public:
    virtual void Init(void);
    virtual int  Run(void);
    virtual void Exit(void);
private:
    bool x_LoadVCFFile(const string& fname);
    bool x_LoadSerializedData(const string& fname);
    void x_UpdateProgress(const string& text);
    bool x_ListErrors(objects::IMessageListener* errCont);
 
    TSeqRange x_ParseRange(const string& range_str);
    void x_ReadVCFBlob(const string& fname, vector<char>& vcf_blob);
};
 
void CColumnarVCFReaderApp::Init(void)
{
    unique_ptr<CArgDescriptions> arg_desc(new CArgDescriptions);
    arg_desc->SetUsageContext(GetArguments().GetProgramBasename(), "Demo application to read VCF files");
 
    arg_desc->AddOptionalKey("i", "InFile", "Input VCF file", CArgDescriptions::eInputFile);
 
    arg_desc->AddOptionalKey("iserial", "InFile", "Input File containing the serialized data", CArgDescriptions::eInputFile);
 
    arg_desc->AddOptionalKey("o", "OutFile", "Single Output File", CArgDescriptions::eOutputFile);
 
    arg_desc->AddFlag("sv_cols", "Output only the first five columns from the data section of the input file");
 
    // for serializing the entire list of variants
    arg_desc->AddOptionalKey("serial", "OutputFile", "Output file for serialized data", CArgDescriptions::eOutputFile);
    
    // to deserialize and compare
    arg_desc->AddFlag("deserialize_and_compare", "Deserialize data and compare it with the existing variant data");
    
    arg_desc->AddOptionalKey("range", "GenomicRange", "Specify the genomic range for which to extract the data", CArgDescriptions::eString);
    // for serializing individual columns
    //arg_desc->AddOptionalKey("serialize_cols", "OutputFile", "Output file with logging serialization and deserialization", CArgDescriptions::eOutputFile);
 
    arg_desc->AddOptionalKey("index", "OutFile", "Output file for listing index vectors", CArgDescriptions::eOutputFile);
 
    //arg_desc->AddOptionalKey("stats", "OutputFile", "Output File for listing statistics", CArgDescriptions::eOutputFile);
 
    //arg_desc->AddFlag("hist", "Compute histogram");
 
    SetupArgDescriptions(arg_desc.release());
}
 
// To serialize the data into one file:
// demo_read_large_vcf -i bbb.gz -serial serial_bbb.bin -o output_bbb.txt -sv_cols
 
// To read the serialized data from a file:
// demo_read_large_vcf -iserial serial_bbb.bin -o output_bbb.txt -sv_cols
 
// demo_read_large_vcf -i bbb.gz -serial serial_bbb.bin -deserialize_and_compare
bool CColumnarVCFReaderApp::x_LoadVCFFile(const string& fname)
{
    cout << "Starting to read file: " << fname << endl;
    CCompressedFile file(fname);
    CColumnarVCFReader reader;
    CRef<CErrorContainer> err_cont{ new CErrorContainer(1000) };
    
    bool has_errors = false;
    bool read_data = false;
    try {
        CColumnarVCFReader::TReportProgress progress = bind(&CColumnarVCFReaderApp::x_UpdateProgress, this, placeholders::_1);
        read_data = reader.ReadData(file.GetIstream(false), nullptr, err_cont, progress);
        has_errors = x_ListErrors(err_cont);
    }
    catch (const CException& e) {
        cout << "Loading VCF file " << fname << " failed: " << e.GetMsg();
        has_errors |= x_ListErrors(err_cont);
    }
    catch (const exception& e) {
        cout << "Loading VCF file " << fname << " failed: " << e.what();
        has_errors |= x_ListErrors(err_cont);
    }
    
    if (!read_data)
        return false;
 
    cout << "VCF file was successfully loaded into memory" << endl;
 
    vector<string> chr_names = reader.GetChromosomeNames();
    for (const auto& it : chr_names) {
        auto var = reader.GetVariantsForChr(it);
        cout << it << "\t" << var->Count() << " variants\n";
    }
 
    const CArgs& args = GetArgs();
    
    if (args["serial"]) {
        const string& serial_fname = args["serial"].AsString();
        auto start = chrono::steady_clock::now();
        for (const auto& it : chr_names) {
            auto var = reader.GetVariantsForChr(it);
            var->WriteSerializedData(it + "_" + serial_fname);
        }
        auto diff = chrono::steady_clock::now() - start;
        cout << "Serialization of all data took: " <<
            chrono::duration_cast<chrono::milliseconds>(diff).count() << " ms" << endl;
 
        if (args["deserialize_and_compare"]) {
            for (const auto& it : chr_names) {
 
                auto orig_var = reader.GetVariantsForChr(it);
                CRef<CVCFVariantList> new_variant(new CVCFVariantList(it, it + "_" + serial_fname));
                if (*orig_var == *new_variant) {
                    cout << it << "\t" << "are the same\n";
                }
                else {
                    cout << it << "\t" << "are not the same\n";
                    return false;
                }
            }
        }
    }
 
    if (args["o"]) {
        try {
            for (const auto& it : chr_names) {
                auto var = reader.GetVariantsForChr(it);
                string out_fname = it + "_" + args["o"].AsString();
                CNcbiOfstream out(out_fname.data());
                var->List(out, args["sv_cols"]);
            }
            cout << "Finished listing columns" << endl;
        }
        catch (const CException& e) {
            cout << "Listing data columns has failed: " << e.GetMsg() << endl;
            return false;
        }
    }
 
    if (args["index"]) {
        try {
            for (const auto& it : chr_names) {
                auto var = reader.GetVariantsForChr(it);
                string out_fname = it + "_" + args["index"].AsString();
                CNcbiOfstream out(out_fname.data());
                var->ListPositionVectors(out);
            }
            cout << "Finished listing index vectors " << endl;
        }
        catch (const CException& e) {
            cout << "Listing index vectors has failed: " << e.GetMsg() << endl;
            return false;
        }
    }
 
    return true;
}
 
void CColumnarVCFReaderApp::x_UpdateProgress(const string& text)
{
    cout << text << endl;
}
 
bool CColumnarVCFReaderApp::x_ListErrors(objects::IMessageListener* errCont)
{
    bool has_errors = (errCont && errCont->Count() > 0);
    if (has_errors) {
        for (size_t i = 0; i < errCont->Count(); ++i) {
            const auto& lerror = errCont->GetError(i);
            cerr << "Line " << lerror.Line() << ". " << lerror.Message() << endl;
        }
    }
    return has_errors;
}
 
bool CColumnarVCFReaderApp::x_LoadSerializedData(const string& fname)
{
    
    // The VCF blob does not contain the name of the chromosome. For now we use a 
    // placeholder for it: 'chr1".
    const CArgs& args = GetArgs();
 
    if (!args["range"]) {
        CRef<CVCFVariantList> new_variant;
        try {
            new_variant.Reset(new CVCFVariantList("chr1", fname));
            cout << "Number of variants read: " << new_variant->Count() << endl;
        }
        catch (const CException& e) {
            cout << "Failed to load data from " << fname << ": " << e.GetMsg() << endl;
        }
        if (!new_variant)
            return false;
 
        return true;
    }
 
    
 
    if (args["range"]) {
        TSeqRange range = x_ParseRange(args["range"].AsString());
        if (range.Empty())
            return true;
 
        vector<char> vcf_blob;
        x_ReadVCFBlob(fname, vcf_blob);
 
        set<string> data_cols(CVCFVariantsBase::s_GetAllColNames().begin(), CVCFVariantsBase::s_GetAllColNames().end());
        CVCFSlicedVariants variants(vcf_blob, &range, data_cols);
 
        cout << "There are " << variants.Count(range) << " variants in the specified range";
 
        string ofname("range_" + NStr::UInt8ToString(range.GetFrom()) + "_" + NStr::UInt8ToString(range.GetTo()) + ".txt");
        CNcbiOfstream out(ofname.data());
        variants.List(out);
    }
    return true;
}
 
TSeqRange CColumnarVCFReaderApp::x_ParseRange(const string& range_str)
{
    TSeqRange range;
    if (range_str.empty())
        return range;
 
    vector<string> positions;
    NStr::Split(range_str, "-", positions, NStr::fSplit_MergeDelimiters);
    if (positions.size() != 2) {
        // don't process bad input data
        return range;
    }
 
    NStr::TruncateSpacesInPlace(positions[0]);
    NStr::TruncateSpacesInPlace(positions[1]);
    TSeqPos from = 1, to = 1;
    try {
        from = NStr::StringToUInt8(positions[0]);
        to = NStr::StringToUInt8(positions[1]);
    }
    catch (const CException&) {
        return range;
    }
 
    cout << "Parsed: " << from << " and " << to << endl;
    range.Set(from, to);
    cout << "Range: " << range.GetFrom() << " and " << range.GetTo() << endl;
    return range;
}
 
void CColumnarVCFReaderApp::x_ReadVCFBlob(const string& fname, vector<char>& vcf_blob)
{
    CFileIO fio;
    try {
        fio.Open(fname, CFileIO::eOpen, CFileIO::eRead);
    }
    catch (const CException& e) {
        cerr << "Cannot open " + fname + "\nas: " + e.GetMsg() << endl;
    }
 
    vcf_blob.resize(0);
    vcf_blob.reserve(fio.GetFileSize());
    char* buf = vcf_blob.data();
    fio.Read(buf, fio.GetFileSize());
    fio.Close();
}
 
 
int CColumnarVCFReaderApp::Run(void)
{
    const CArgs& args = GetArgs();
 
    if (args["i"]) {
        if (!x_LoadVCFFile(args["i"].AsString())) {
            return 1;
        }
    }
    else if (args["iserial"]) {
        if (!x_LoadSerializedData(args["iserial"].AsString())) {
            return 1;
        }
    }
    return 0;
}
 
/*
 
    if (args["i"]) {
       if (args["serialize_cols"]) {
            reader.SerializeToDisk(fname, &args["serialize_cols"].AsOutputFile());
            reader.Deserialize(fname, &args["serialize_cols"].AsOutputFile());
            cout << "Finished serializing data and deserialization was successful" << endl;
        }
 
        if (args["stats"]) {
            reader.GetStatistics(args["stats"].AsOutputFile());
            cout << "Finished printing statistics" << endl;
            try {
            }
            catch (const CException& e) {
                cout << "Printing statistics of vectors has failed" << endl;
                return 4;
            }
        }
    }
    
*/
 
 
void CColumnarVCFReaderApp::Exit(void)
{
    SetDiagStream(0);
#ifdef NCBI_OS_MSWIN
    cout << "Press any key to end the demo app" << endl;
    getchar();
#endif
}
 
 
/////////////////////////////////////////////////////////////////////////////
//  MAIN
int NcbiSys_main(int argc, ncbi::TXChar* argv[])
{
    // Execute main application function
    return CColumnarVCFReaderApp().AppMain(argc, argv);
}