unit_test_gff3flybasewriter.cpp

Go to the documentation of this file.

Go to the SVN repository for this file.

/*  $Id: unit_test_gff3flybasewriter.cpp 93574 2021-04-30 16:19:19Z stakhovv $
* ===========================================================================
*
*                            PUBLIC DOMAIN NOTICE
*               National Center for Biotechnology Information
*
*  This software/database is a "United States Government Work" under the
*  terms of the United States Copyright Act.  It was written as part of
*  the author's official duties as a United States Government employee and
*  thus cannot be copyrighted.  This software/database is freely available
*  to the public for use. The National Library of Medicine and the U.S.
*  Government have not placed any restriction on its use or reproduction.
*
*  Although all reasonable efforts have been taken to ensure the accuracy
*  and reliability of the software and data, the NLM and the U.S.
*  Government do not and cannot warrant the performance or results that
*  may be obtained by using this software or data. The NLM and the U.S.
*  Government disclaim all warranties, express or implied, including
*  warranties of performance, merchantability or fitness for any particular
*  purpose.
*
*  Please cite the author in any work or product based on this material.
*
* ===========================================================================
*
* Author:  Mostly Mike Kornbluh, NCBI.
*          Customizations by Frank Ludwig, NCBI.
*
* File Description:
*   GFF3 writer unit test.
*
* ===========================================================================
*/
 
#include <ncbi_pch.hpp>
 
#include <corelib/ncbi_system.hpp>
#include <corelib/ncbiapp.hpp>
#include <corelib/ncbifile.hpp>
#include <connect/ncbi_core_cxx.hpp>
#include <serial/serial.hpp>
#include <serial/objistr.hpp>
#include <serial/streamiter.hpp>
 
#include <objmgr/object_manager.hpp>
#include <objmgr/scope.hpp>
#include <objmgr/seq_entry_ci.hpp>
#include <objmgr/bioseq_ci.hpp>
#include <objmgr/annot_ci.hpp>
#include <objtools/data_loaders/genbank/gbloader.hpp>
#include <dbapi/driver/drivers.hpp>
 
#include <objtools/writers/writer_exception.hpp>
#include <objtools/writers/gff3flybase_writer.hpp>
#include "error_logger.hpp"
 
#include <cstdio>
 
// This header must be included before all Boost.Test headers if there are any
#include <corelib/test_boost.hpp>
 
USING_NCBI_SCOPE;
USING_SCOPE(objects);
 
//  ============================================================================
//  Customization data:
const string extInput("asn");
const string extDatabase("database");
const string extOutput("gff3");
const string extErrors("errors");
const string extKeep("new");
 
const string dirTestFiles("gff3flybasewriter_test_cases");
// !!!
// !!! Must also customize reader type in sRunTest !!!
// !!!
//  ============================================================================
 
struct STestInfo {
    CFile mInFile;
    CFile mOutFile;
    CFile mErrorFile;
    CFile mDatabaseFile;
    string mObjType;
};
typedef string TTestName;
typedef map<TTestName, STestInfo> TTestNameToInfoMap;
 
class CTestNameToInfoMapLoader {
public:
    CTestNameToInfoMapLoader(
        TTestNameToInfoMap * pTestNameToInfoMap,
        const string& extInput,
        const string& extOutput,
        const string& extErrors,
        const string& extDatabase)
        : m_pTestNameToInfoMap(pTestNameToInfoMap),
          mExtInput(extInput),
          mExtOutput(extOutput),
          mExtErrors(extErrors),
          mExtDatabase(extDatabase)
    { }
 
    void operator()( const CDirEntry & dirEntry ) {
 
        if( ! dirEntry.IsFile() ) {
            return;
        }
 
        CFile file(dirEntry);
        string name = file.GetName();
        if (NStr::EndsWith(name, ".txt")  ||  NStr::StartsWith(name, ".")) {
            return;
        }
        if (NStr::EndsWith(name, extKeep)) {
            return;
        }
 
        // extract info from the file name
        const string sFileName = file.GetName();
        vector<string> vecFileNamePieces;
        NStr::Split( sFileName, ".", vecFileNamePieces );
        BOOST_REQUIRE(vecFileNamePieces.size() == 2);
 
        string sTestName = vecFileNamePieces[0];
        BOOST_REQUIRE(!sTestName.empty());
        string sFileType = vecFileNamePieces[1];
        BOOST_REQUIRE(!sFileType.empty());
 
        STestInfo & test_info_to_load =
            (*m_pTestNameToInfoMap)[vecFileNamePieces[0]];
        test_info_to_load.mObjType = "align";
 
        // figure out what type of file we have and set appropriately
        if (sFileType == mExtInput) {
            BOOST_REQUIRE( test_info_to_load.mInFile.GetPath().empty() );
            test_info_to_load.mInFile = file;
        }
        else if (sFileType == mExtDatabase) {
            BOOST_REQUIRE( test_info_to_load.mDatabaseFile.GetPath().empty() );
            test_info_to_load.mDatabaseFile = file;
        }
        else if (sFileType == mExtOutput) {
            BOOST_REQUIRE( test_info_to_load.mOutFile.GetPath().empty() );
            test_info_to_load.mOutFile = file;
        }
        else if (sFileType == mExtErrors) {
            BOOST_REQUIRE( test_info_to_load.mErrorFile.GetPath().empty() );
            test_info_to_load.mErrorFile = file;
        }
 
        else {
            BOOST_FAIL("Unknown file type " << sFileName << ".");
        }
    }
 
private:
    // raw pointer because we do NOT own this
    TTestNameToInfoMap * m_pTestNameToInfoMap;
    string mExtInput;
    string mExtOutput;
    string mExtErrors;
    string mExtDatabase;
};
 
CGff3FlybaseWriter* sGetWriter(CScope& scope, CNcbiOstream& ostr)
{
    return new CGff3FlybaseWriter(scope, ostr);
}
 
void sUpdateCase(CDir& test_cases_dir, const string& test_name)
{
    string input = CDir::ConcatPath( test_cases_dir.GetPath(), test_name + "." + extInput);
    string database = CDir::ConcatPath( test_cases_dir.GetPath(), test_name + "." + extDatabase);
    string output = CDir::ConcatPath( test_cases_dir.GetPath(), test_name + "." + extOutput);
    string errors = CDir::ConcatPath( test_cases_dir.GetPath(), test_name + "." + extErrors);
    if (!CFile(input).Exists()) {
         BOOST_FAIL("input file " << input << " does not exist.");
    }
    //string test_base, test_type;
    //NStr::SplitInTwo(test_name, ".", test_base, test_type);
    cerr << "Creating new test case from " << input << " ..." << endl;
 
    CErrorLogger logger(errors);
 
    //get a scope
    CRef<CObjectManager> pObjMngr = CObjectManager::GetInstance();
   // CGBDataLoader::RegisterInObjectManager(*pObjMngr).GetLoader()->SetAlwaysLoadExternal(false);
   // pScope->AddDefaults();
 
    //get a writer object
    CNcbiIfstream ifstr(input.c_str(), ios::binary);
    unique_ptr<CObjectIStream> pI(CObjectIStream::Open(eSerial_AsnText, ifstr));
 
    CNcbiIfstream dbstr(database.c_str(), ios::binary);
    unique_ptr<CObjectIStream> pDB(CObjectIStream::Open(eSerial_AsnText, dbstr));
    CRef<CScope> pScope(new CScope(*pObjMngr));
    {
        CObjectIStreamIterator<CSeq_entry> entry_it(*pDB);
        for (const auto& entry : entry_it) {
            pScope->AddTopLevelSeqEntry(entry);
        }
    }
 
    CNcbiOfstream ofstr(output.c_str());
    CGff3FlybaseWriter* pWriter = sGetWriter(*pScope, ofstr);
 
    CRef<CSeq_align> pAlign(new CSeq_align);
    *pI >> *pAlign;
    pWriter->WriteHeader();
    pWriter->WriteAlign(*pAlign);
    pWriter->WriteFooter();
    while (!pI->EndOfData()) {
        *pI >> *pAlign;
        pWriter->WriteHeader();
        pWriter->WriteAlign(*pAlign);
        pWriter->WriteFooter();
    }
    delete pWriter;
    ofstr.flush();
 
    ifstr.close();
    ofstr.close();
 
    cerr << "    Produced new flybase file " << output << "." << endl;
    cerr << " ... Done." << endl;
}
 
//  ----------------------------------------------------------------------------
void sUpdateAll(CDir& test_cases_dir)
//  ----------------------------------------------------------------------------
{
    const vector<string> kEmptyStringVec;
    TTestNameToInfoMap testNameToInfoMap;
    CTestNameToInfoMapLoader testInfoLoader(
        &testNameToInfoMap, extInput, extOutput, extErrors, extDatabase);
    FindFilesInDir(
        test_cases_dir,
        kEmptyStringVec,
        kEmptyStringVec,
        testInfoLoader,
        fFF_Default | fFF_Recursive );
 
    ITERATE(TTestNameToInfoMap, name_to_info_it, testNameToInfoMap) {
        const string & sName = name_to_info_it->first;
          //  "." + name_to_info_it->second.mObjType;
        sUpdateCase(test_cases_dir, sName);
    }
}
 
//  ----------------------------------------------------------------------------
void sRunTest(const string &sTestName, const STestInfo & testInfo, bool keep)
//  ----------------------------------------------------------------------------
{
    cerr << "Testing " << testInfo.mInFile.GetName() << " and " <<
        testInfo.mDatabaseFile.GetName() << " "
        << " against " <<
        testInfo.mOutFile.GetName() << " and " <<
        testInfo.mErrorFile.GetName() << endl;
 
    string logName = CDirEntry::GetTmpName();
    CErrorLogger logger(logName);
 
    //get a scope
    CRef<CObjectManager> pObjMngr = CObjectManager::GetInstance();
 //   CGBDataLoader::RegisterInObjectManager(*pObjMngr).GetLoader()->SetAlwaysLoadExternal(false);
  //  pScope->AddDefaults();
 
    //get a writer object
    CNcbiIfstream ifstr(testInfo.mInFile.GetPath().c_str(), ios::binary);
    unique_ptr<CObjectIStream> pI(CObjectIStream::Open(eSerial_AsnText, ifstr));
 
    CNcbiIfstream dbstr(testInfo.mDatabaseFile.GetPath().c_str(), ios::binary);
    unique_ptr<CObjectIStream> pDB(CObjectIStream::Open(eSerial_AsnText, dbstr));
    CRef<CScope> pScope(new CScope(*pObjMngr));
    {
        CObjectIStreamIterator<CSeq_entry> entry_it(*pDB);
        for (const auto& entry :entry_it) {
            pScope->AddTopLevelSeqEntry(entry);
        }
    }
    string resultName = CDirEntry::GetTmpName();
    CNcbiOfstream ofstr(resultName.c_str());
    CGff3FlybaseWriter* pWriter = sGetWriter(*pScope, ofstr);
 
    CRef<CSeq_align> pAlign(new CSeq_align);
    *pI >> *pAlign;
    pWriter->WriteHeader();
    pWriter->WriteAlign(*pAlign);
    pWriter->WriteFooter();
    while (!pI->EndOfData()) {
        *pI >> *pAlign;
        pWriter->WriteHeader();
        pWriter->WriteAlign(*pAlign);
        pWriter->WriteFooter();
    }
    delete pWriter;
    ofstr.flush();
    ifstr.close();
    ofstr.close();
 
    bool success = testInfo.mOutFile.CompareTextContents(resultName, CFile::eIgnoreWs);
    if (!success) {
        CDirEntry deResult = CDirEntry(resultName);
        if (keep) {
            deResult.Copy(testInfo.mOutFile.GetPath() + "." + extKeep,
                CDirEntry::fCF_Overwrite);
        }
        deResult.Remove();
        CDirEntry(logName).Remove();
        BOOST_ERROR("Error: " << sTestName << " failed due to post processing diffs.");
    }
    CDirEntry(resultName).Remove();
 
    success = testInfo.mErrorFile.CompareTextContents(logName, CFile::eIgnoreWs);
    CDirEntry deErrors = CDirEntry(logName);
    if (!success  &&  keep) {
        deErrors.Copy(testInfo.mErrorFile.GetPath() + "." + extKeep);
    }
    deErrors.Remove();
    if (!success) {
        BOOST_ERROR("Error: " << sTestName << " failed due to error handling diffs.");
    }
};
NCBITEST_AUTO_INIT()
{
}
 
NCBITEST_INIT_CMDLINE(arg_descrs)
{
    arg_descrs->AddDefaultKey("test-dir", "TEST_FILE_DIRECTORY",
        "Set the root directory under which all test files can be found.",
        CArgDescriptions::eDirectory,
        dirTestFiles );
    arg_descrs->AddDefaultKey("update-case", "UPDATE_CASE",
        "Produce .asn and .error files from given name for new or updated test case.",
        CArgDescriptions::eString,
        "" );
    arg_descrs->AddFlag("update-all",
        "Update all test cases to current reader code (dangerous).",
        true );
    arg_descrs->AddFlag("keep-diffs",
        "Keep output files that are different from the expected.",
        true );
}
 
NCBITEST_AUTO_FINI()
{
}
 
BOOST_AUTO_TEST_CASE(RunTests)
{
    const CArgs& args = CNcbiApplication::Instance()->GetArgs();
 
    CDir test_cases_dir( args["test-dir"].AsDirectory() );
    BOOST_REQUIRE_MESSAGE( test_cases_dir.IsDir(),
        "Cannot find dir: " << test_cases_dir.GetPath() );
 
    bool update_all = args["update-all"].AsBoolean();
    if (update_all) {
        sUpdateAll(test_cases_dir);
        return;
    }
 
    string update_case = args["update-case"].AsString();
    if (!update_case.empty()) {
        sUpdateCase(test_cases_dir, update_case);
        return;
    }
 
    const vector<string> kEmptyStringVec;
    TTestNameToInfoMap testNameToInfoMap;
    CTestNameToInfoMapLoader testInfoLoader(
        &testNameToInfoMap, extInput, extOutput, extErrors, extDatabase);
    FindFilesInDir(
        test_cases_dir,
        kEmptyStringVec,
        kEmptyStringVec,
        testInfoLoader,
        fFF_Default | fFF_Recursive );
 
    ITERATE(TTestNameToInfoMap, name_to_info_it, testNameToInfoMap) {
        const string & sName = name_to_info_it->first;
        const STestInfo & testInfo = name_to_info_it->second;
        cout << "Verifying: " << sName << endl;
        BOOST_REQUIRE_MESSAGE( testInfo.mInFile.Exists(),
            extInput + " file does not exist: " << testInfo.mInFile.GetPath() );
        BOOST_REQUIRE_MESSAGE( testInfo.mDatabaseFile.Exists(),
            extDatabase + " file does not exist: " << testInfo.mInFile.GetPath() );
        BOOST_REQUIRE_MESSAGE( testInfo.mOutFile.Exists(),
            extOutput + " file does not exist: " << testInfo.mOutFile.GetPath() );
        BOOST_REQUIRE_MESSAGE( testInfo.mErrorFile.Exists(),
            extErrors + " file does not exist: " << testInfo.mErrorFile.GetPath() );
    }
    ITERATE(TTestNameToInfoMap, name_to_info_it, testNameToInfoMap) {
        const string & sName = name_to_info_it->first;
        const STestInfo & testInfo = name_to_info_it->second;
 
        cout << "Running test: " << sName << endl;
 
        BOOST_CHECK_NO_THROW(sRunTest(sName, testInfo, args["keep-diffs"]));
    }
}