newcleanupp.cpp

Go to the documentation of this file.

Go to the SVN repository for this file.

/*
* ===========================================================================
*
*                            PUBLIC DOMAIN NOTICE
*               National Center for Biotechnology Information
*
*  This software/database is a "United States Government Work" under the
*  terms of the United States Copyright Act.  It was written as part of
*  the author's official duties as a United States Government employee and
*  thus cannot be copyrighted.  This software/database is freely available
*  to the public for use. The National Library of Medicine and the U.S.
*  Government have not placed any restriction on its use or reproduction.
*
*  Although all reasonable efforts have been taken to ensure the accuracy
*  and reliability of the software and data, the NLM and the U.S.
*  Government do not and cannot warrant the performance or results that
*  may be obtained by using this software or data. The NLM and the U.S.
*  Government disclaim all warranties, express or implied, including
*  warranties of performance, merchantability or fitness for any particular
*  purpose.
*
*  Please cite the author in any work or product based on this material.
*
* ===========================================================================
*
* Author: Robert Smith, Jonathan Kans, Michael Kornbluh
*
* File Description:
*   Basic and Extended Cleanup of CSeq_entries, etc.
*
* ===========================================================================
*/
 
// All this functionality is packed into this one file for ease of
// searching.  If it gets big enough, it will be broken up in the future.
 
#include <ncbi_pch.hpp>
 
#include <corelib/ncbi_autoinit.hpp>
 
#include <objects/misc/sequence_macros.hpp>
#include <objmgr/annot_ci.hpp>
#include <objmgr/feat_ci.hpp>
#include <objmgr/seqdesc_ci.hpp>
#include <objmgr/scope.hpp>
#include <objmgr/seq_vector.hpp>
#include <objmgr/util/seq_loc_util.hpp>
#include <objmgr/util/feature.hpp>
#include <objmgr/util/sequence.hpp>
#include <objtools/cleanup/cleanup_change.hpp>
 
#include <objtools/cleanup/cleanup.hpp>
#include "newcleanupp.hpp"
 
#include "cleanup_utils.hpp"
 
#include <objtools/cleanup/cleanup_pub.hpp>
#include <objtools/cleanup/fix_feature_id.hpp>
#include <objtools/readers/read_util.hpp>
 
#include <objmgr/bioseq_ci.hpp>
#include <objmgr/object_manager.hpp>
#include <objmgr/scope.hpp>
 
#include <objects/medline/Medline_entry.hpp>
#include <objects/valid/Comment_rule.hpp>
#include <objects/valid/Comment_set.hpp>
 
#include <util/ncbi_cache.hpp>
#include <util/sequtil/sequtil_convert.hpp>
#include <util/sequtil/sequtil_manip.hpp>
#include <util/xregexp/regexp.hpp>
#include <util/strsearch.hpp>
 
#include <objmgr/util/objutil.hpp>
 
#include "autogenerated_cleanup.hpp"
#include "autogenerated_extended_cleanup.hpp"
 
BEGIN_NCBI_SCOPE
BEGIN_SCOPE(objects)
 
//const int CNewCleanup_imp::NCBI_CLEANUP_VERSION = 1;
 
// We don't want to use CompressSpaces inside the likes of COMPRESS_STRING_MEMBER
// we prefer our own version
#define CompressSpaces x_CompressSpaces
 
namespace {
 
    // a CRegexp that has lock and unlock methods,
    // and also inherits from CObject
    class CRegexpWithLock : public CObject, public CRegexp {
    public:
        CRegexpWithLock( const CTempStringEx & pattern,
            CRegexp::TCompile flags ) : CRegexp(pattern, flags) { }
 
        void Lock(void) { m_mutex.Lock(); }
        void Unlock(void) { m_mutex.Unlock(); }
 
    private:
        CMutex               m_mutex;
    };
    typedef CRef<CRegexpWithLock> TRegexpWithLockRef;
 
    // this protects its inner object by locking
    // it as soon as it's created and unlocking it when destroyed.
    // this way, there's only one working CLockingRef on the object at a time
    template<typename TLockableObj>
    class CLockingRef {
    public:
        explicit
        CLockingRef(TLockableObj *pLockableObj) :
        m_pLockableObj(pLockableObj)
        {
            m_pLockableObj->Lock();
        }
 
        ~CLockingRef(void) {
            m_pLockableObj->Unlock();
        }
 
        TLockableObj * operator->(void) { return m_pLockableObj.GetPointer(); }
 
    private:
        CRef<TLockableObj> m_pLockableObj;
    };
    typedef CLockingRef<CRegexpWithLock> CCachedRegexp;
 
    // careful! the key is compared as a *pointer*, NOT via
    // strcmp or anything like that.  For safety, just use
    // string literals.
    typedef pair<const char *, CRegexp::TCompile> TRegexpKey;
    typedef TRegexpWithLockRef TRegexpValue;
 
    class CRegexpCacheHandler :
        public CCacheElement_Handler<TRegexpKey, TRegexpValue>
    {
    public:
        TRegexpValue CreateValue(const TRegexpKey & regexp_key )
        {
            return Ref(new CRegexpWithLock(
                regexp_key.first, regexp_key.second));
        }
    };
 
    class CRegexpCache {
    public:
 
        CRegexpCache(void)
            : m_Cache(100) { }
 
        CCachedRegexp Get( const char * pattern,
            CRegexp::TCompile flags = CRegexp::fCompile_default )
        {
            TRegexpKey regexpKey(pattern, flags);
            TRegexpWithLockRef regexpLockRef = m_Cache[regexpKey];
            return CCachedRegexp(regexpLockRef.GetPointer());
        }
 
    private:
        typedef CCache<TRegexpKey, TRegexpValue,
            CRegexpCacheHandler> TUnderlyingCache;
        TUnderlyingCache m_Cache;
    };
 
    // the actual cache
    CRegexpCache regexpCache;
}
 
// Constructor
CNewCleanup_imp::CNewCleanup_imp (CRef<CCleanupChange> changes, Uint4 options)
    : m_Changes(changes),
      m_Options(options)
{
    if (options & CCleanup::eClean_GpipeMode) {
        m_IsGpipe = true;
    }
 
    if (options & CCleanup::eClean_SyncGenCodes) {
        m_SyncGenCodes = true;
    }
 
    if (options & CCleanup::eClean_KeepTopSet) {
        m_KeepTopNestedSet = true;
    }
 
    if (options & CCleanup::eClean_KeepSingleSeqSet) {
        m_KeepSingleSeqSet = true;
    }
 
    m_Objmgr = CObjectManager::GetInstance ();
    m_Scope.Reset (new CScope (*m_Objmgr));
 
}
 
// Destructor
CNewCleanup_imp::~CNewCleanup_imp (void)
 
{
}
 
// Main methods
 
void CNewCleanup_imp::BasicCleanupSeqEntry (
    CSeq_entry& se
)
 
{
    // The class CAutogeneratedCleanup is actually auto-generated code
    // created by datatool from autogenerated_cleanup.txt
    // It traverses into the CSeq_entry object we have here and
    // calls our functions here.
    // The idea is that we don't have to hand-write the
    // error-prone traversal code.
    SetGlobalFlags(se);
    CAutogeneratedCleanup auto_cleanup( *m_Scope, *this );
    auto_cleanup.BasicCleanupSeqEntry( se );
    x_PostProcessing();
 
    EXPLORE_ALL_BIOSEQS_WITHIN_SEQENTRY (bit, se) {
        CBioseq& bs = *bit;
        SetGeneticCode (bs);
    }
}
 
//LCOV_EXCL_START
//not used by asn_cleanup because we clean the submit block separately
//and use read hooks for the seq-entries
void CNewCleanup_imp::BasicCleanupSeqSubmit (
    CSeq_submit& ss
)
 
{
    SetGlobalFlags(ss);
    CAutogeneratedCleanup auto_cleanup( *m_Scope, *this );
    auto_cleanup.BasicCleanupSeqSubmit( ss );
    x_PostProcessing();
 
    CRef<CSeq_entry> se (ss.SetData().SetEntrys().front());
    if (se.NotEmpty()) {
        EXPLORE_ALL_BIOSEQS_WITHIN_SEQENTRY (bit, *se) {
            CBioseq& bs = *bit;
            SetGeneticCode (bs);
        }
    }
}
//LCOV_EXCL_STOP
 
 
void CNewCleanup_imp::BasicCleanupSubmitblock(CSubmit_block& block)
 
{
    SubmitblockBC(block);
}
 
 
void CNewCleanup_imp::BasicCleanupSeqAnnot (
    CSeq_annot& sa
)
 
{
    // no Seq-entry context, so skip setup function
    ResetGlobalFlags();
    CAutogeneratedCleanup auto_cleanup( *m_Scope, *this );
    auto_cleanup.BasicCleanupSeqAnnot( sa );
    x_PostProcessing();
}
 
void CNewCleanup_imp::BasicCleanupBioseq(
    CBioseq& bs
)
 
{
    // no Seq-entry context, so skip setup function
    SetGlobalFlags(bs);
    CAutogeneratedCleanup auto_cleanup( *m_Scope, *this );
    auto_cleanup.BasicCleanupBioseq( bs );
 
    x_PostProcessing();
 
    SetGeneticCode (bs);
}
 
void CNewCleanup_imp::BasicCleanupBioseqSet (
    CBioseq_set& bss
)
 
{
    // no Seq-entry context, so skip setup function
    SetGlobalFlags(bss);
    CAutogeneratedCleanup auto_cleanup( *m_Scope, *this );
    auto_cleanup.BasicCleanupBioseqSet( bss );
    x_PostProcessing();
 
    EXPLORE_ALL_BIOSEQS_WITHIN_SEQSET (bit, bss) {
        CBioseq& bs = *bit;
        SetGeneticCode (bs);
    }
}
 
void CNewCleanup_imp::BasicCleanupSeqFeat (
    CSeq_feat& sf
)
 
{
    // no Seq-entry context, so skip setup function
    ResetGlobalFlags();
    CAutogeneratedCleanup auto_cleanup( *m_Scope, *this );
    auto_cleanup.BasicCleanupSeqFeat( sf );
    x_PostProcessing();
}
 
 
void CNewCleanup_imp::BasicCleanupBioSource (
    CBioSource& src
)
 
{
    // no Seq-entry context, so skip setup function
    ResetGlobalFlags();
    CAutogeneratedCleanup auto_cleanup( *m_Scope, *this );
    CRef<CSeq_feat> f(new CSeq_feat());
    f->SetData().SetBiosrc().Assign(src);
    auto_cleanup.BasicCleanupSeqFeat(*f);
    x_PostProcessing();
    src.Assign(f->GetData().GetBiosrc());
}
 
 
void CNewCleanup_imp::ExtendedCleanup(CBioSource& src)
{
    BiosourceBC(src);
    BioSourceEC(src);
}
 
 
void CNewCleanup_imp::BasicCleanupSeqEntryHandle (
    CSeq_entry_Handle& seh
)
{
    CConstRef<CSeq_entry> seq_entry = seh.GetCompleteSeq_entry();
    CSeq_entry* se = const_cast<CSeq_entry*>(seq_entry.GetPointer());
    BasicCleanupSeqEntry(*se);
}
 
void CNewCleanup_imp::BasicCleanupBioseqHandle (
    CBioseq_Handle& bsh
)
{
    BasicCleanupBioseq(*const_cast<CBioseq*>(bsh.GetCompleteBioseq().GetPointer()));
}
 
void CNewCleanup_imp::BasicCleanupBioseqSetHandle (
    CBioseq_set_Handle& bssh
)
{
    BasicCleanupBioseqSet(*const_cast<CBioseq_set*>(bssh.GetCompleteBioseq_set().GetPointer()));
}
 
void CNewCleanup_imp::BasicCleanupSeqAnnotHandle (
    CSeq_annot_Handle& sah
)
{
    // clean a copy, and then update via the edit handle
 
    CRef<CSeq_annot> new_seq_annot( new CSeq_annot );
    new_seq_annot->Assign( *sah.GetCompleteSeq_annot() );
 
    CSeq_annot_EditHandle edit_handle = sah.GetEditHandle();
 
    BasicCleanupSeqAnnot( *new_seq_annot );
 
    // Since CSeq_annot_EditHandle doesn't have ".Set[Fld]()" methods or
    // a Replace() method, it's a little more tricky than the others.
    CSeq_entry_EditHandle annot_parent = edit_handle.GetParentEntry();
    if( annot_parent ) {
        edit_handle.Remove();
        sah = annot_parent.AttachAnnot( *new_seq_annot );
    } else {
        // if not part of anything else, a simple swap will do
        CSeq_annot_Handle new_sah = m_Scope->AddSeq_annot( *new_seq_annot );
        edit_handle.Swap( new_sah );
    }
}
 
void CNewCleanup_imp::BasicCleanupSeqFeatHandle (
    CSeq_feat_Handle& sfh
)
{
    // clean a copy, and then update via the edit handle
 
    CRef<CSeq_feat> new_seq_feat( new CSeq_feat );
    new_seq_feat->Assign( *sfh.GetOriginalSeq_feat() );
 
    CSeq_feat_EditHandle edit_handle( sfh );
 
    BasicCleanupSeqFeat( *new_seq_feat );
 
    edit_handle.Replace( *new_seq_feat );
}
 
 
void CNewCleanup_imp::BasicCleanup(CPubdesc& pd, bool strip_serial)
{
    bool was_strip_serial = m_StripSerial;
    m_StripSerial = strip_serial;
    PubdescBC(pd);
    m_StripSerial = was_strip_serial;
}
 
 
void CNewCleanup_imp::BasicCleanup(CSeqdesc& desc)
{
    ResetGlobalFlags();
    CAutogeneratedCleanup auto_cleanup(*m_Scope, *this);
    auto_cleanup.BasicCleanupSeqdesc(desc);
    x_PostProcessing();
}
 
 
 
 
// Implementation methods
 
void CNewCleanup_imp::SetGeneticCode (
    CBioseq& bs
)
 
{
    if ( ! m_SyncGenCodes ) return;
 
    CBioseq_Handle bsh = m_Scope->GetBioseqHandle(bs);
    if (!bsh) return;
 
    if (CCleanup::SetGeneticCodes(bsh)) {
        ChangeMade(CCleanupChange::eChangeGeneticCode);
    }
}
 
void CNewCleanup_imp::ChangeMade (CCleanupChange::EChanges e)
{
    if (m_Changes) {
        m_Changes->SetChanged (e);
    }
}
 
void CNewCleanup_imp::EnteringEntry (
    CSeq_entry& se
)
 
{
#if 0
    SSeqEntryInfo seqEntryInfo;
    if( ! m_SeqEntryInfoStack.empty() ) {
        // inherit from parent by default
        seqEntryInfo = m_SeqEntryInfoStack.top();
    } else {
        seqEntryInfo.m_IsEmblOrDdbj = false;
        seqEntryInfo.m_StripSerial = true;
    }
#endif
 
    // for cleanup Seq-entry and Seq-submit, set scope and parentize.
    // We use exceptions for AddTopLevelSeqEntry because we need to detect
    // if we've already processed the given Seq-entry.
    {{
         CSeq_entry_Handle seh =
             m_Scope->GetSeq_entryHandle(se, CScope::eMissing_Null);
         if (seh) {
#if 0
             // all code paths in this function must result
             // in m_SeqEntryInfoStack getting a "push"
             m_SeqEntryInfoStack.push( m_SeqEntryInfoStack.top() );
#endif
             return;
         }
 
         m_Scope->AddTopLevelSeqEntry (se);
         se.Parentize();
     }}
 
#if 0
    // a few differences based on sequence identifier type
    // (some values are reset here because they shouldn't inherit
    // from higher seq-entry's)
    VISIT_ALL_BIOSEQS_WITHIN_SEQENTRY (bs_itr, se) {
        const CBioseq& bs = *bs_itr;
        FOR_EACH_SEQID_ON_BIOSEQ (sid_itr, bs) {
            const CSeq_id& sid = **sid_itr;
            SWITCH_ON_SEQID_CHOICE (sid) {
                case NCBI_SEQID(Genbank):
                case NCBI_SEQID(Tpg):
                    {
                        const CTextseq_id& tsid = *GET_FIELD (sid, Textseq_Id);
                        if (FIELD_IS_SET (tsid, Accession)) {
                            const string& acc = GET_FIELD (tsid, Accession);
                            if (acc.length() == 6) {
                                seqEntryInfo.m_StripSerial = false;
                            }
                        }
                    }
                    break;
                case NCBI_SEQID(Embl):
                case NCBI_SEQID(Ddbj):
                    seqEntryInfo.m_StripSerial = false;
                    seqEntryInfo.m_IsEmblOrDdbj = true;
                    break;
                case NCBI_SEQID(not_set):
                case NCBI_SEQID(Local):
                case NCBI_SEQID(Other):
                case NCBI_SEQID(General):
                    break;
                case NCBI_SEQID(Gibbsq):
                case NCBI_SEQID(Gibbmt):
                case NCBI_SEQID(Pir):
                case NCBI_SEQID(Swissprot):
                case NCBI_SEQID(Patent):
                case NCBI_SEQID(Prf):
                case NCBI_SEQID(Pdb):
                case NCBI_SEQID(Gpipe):
                case NCBI_SEQID(Tpe):
                case NCBI_SEQID(Tpd):
                    seqEntryInfo.m_StripSerial = false;
                    break;
                default:
                    break;
            }
        }
    }
 
    m_SeqEntryInfoStack.push(seqEntryInfo);
#endif
}
 
void CNewCleanup_imp::LeavingEntry (
    CSeq_entry& se
)
 
{
#if 0
    m_SeqEntryInfoStack.pop();
#endif
}
 
// Strips all spaces in string in following manner. If the function
// meets several spaces (spaces and tabs) in succession it replaces them
// with one space. Strips all spaces after '(' and before ( ')' or ',' ).
void CNewCleanup_imp::x_StripSpacesMarkChanged(string& str)
{
    if (StripSpaces(str)) {
        ChangeMade(CCleanupChange::eTrimSpaces);
    }
}
 
void CNewCleanup_imp::x_RemoveSpacesBetweenTildesMarkChanged( std::string & str )
{
    if( RemoveSpacesBetweenTildes(str) ) {
        ChangeMade(CCleanupChange::eTrimSpaces);
    }
}
 
void CNewCleanup_imp::X_CommentTildeFixes(std::string & str)
{
/*
#ifndef NCBI_OS_MSWIN
    string orig = str;
    NStr::ReplaceInPlace(str, "based on SOLiD3 (Applied Biosystems)~~", "based on SOLiD3 (Applied Biosystems)", false, false);
    NStr::ReplaceInPlace(str, "Biological resourse center, NITE (NRBC)~~", "Biological resourse center, NITE (NRBC)", false, false);
    NStr::ReplaceInPlace(str, "developmental01.html~~", "developmental01.html", false, false);
    NStr::ReplaceInPlace(str, "http://bionano.toyo.ac.jp/~~", "http://bionano.toyo.ac.jp/", false, false);
    NStr::ReplaceInPlace(str, "http://dictycdb1.biol.tsukuba.ac.jp/acytodb/~~", "http://dictycdb1.biol.tsukuba.ac.jp/acytodb/", false, false);
    NStr::ReplaceInPlace(str, "http://egg.umh.es~~", "http://egg.umh.es", false, false);
    NStr::ReplaceInPlace(str, "http://www.aist.go.jp/~~", "http://www.aist.go.jp/", false, false);
    NStr::ReplaceInPlace(str, "http://www.bio.nite.go.jp/~~", "http://www.bio.nite.go.jp/", false, false);
    NStr::ReplaceInPlace(str, "http://www.bio.nite.go.jp/ngac/e/~~", "http://www.bio.nite.go.jp/ngac/e/", false, false);
    NStr::ReplaceInPlace(str, "http://www.brs.kyushu-u.ac.jp/~fcmic/~~", "http://www.brs.kyushu-u.ac.jp/~fcmic/", false, false);
    NStr::ReplaceInPlace(str, "http://www.miyazaki-u.ac.jp/ir/english/index.html~~", "http://www.miyazaki-u.ac.jp/ir/english/index.html", false, false);
    NStr::ReplaceInPlace(str, "URL:http://www.bio.nite.go.jp/ngac/e/~~", "URL:http://www.bio.nite.go.jp/ngac/e/", false, false);
    if (!NStr::Equal(orig, str)) {
        ChangeMade(CCleanupChange::eTrimSpaces);
    }
#endif //NCBI_OS_MSWIN
*/
}
 
void CNewCleanup_imp::x_TruncateSpacesMarkChanged( std::string & str )
{
    const size_t old_str_size = str.length();
    NStr::TruncateSpacesInPlace(str);
    if( old_str_size != str.length() ) {
        ChangeMade(CCleanupChange::eTrimSpaces);
    }
}
 
void CNewCleanup_imp::x_TrimInternalSemicolonsMarkChanged( std::string & str )
{
    const size_t old_str_size = str.length();
    TrimInternalSemicolons(str);
    if( old_str_size != str.length() ) {
        ChangeMade(CCleanupChange::eTrimInternalSemicolons);
    }
}
 
 
 
static void s_IncrementSeqCount(const CBioseq& bioseq, 
        int& num_nucs, int& num_prots)
{
    if (!bioseq.IsSetInst()) {
        return;
    }
 
    const auto& inst = bioseq.GetInst();
    if (inst.IsNa()) {
        ++num_nucs;
    } else if (inst.IsAa()) {
        ++num_prots;
    }
}
 
 
static bool s_IsValidNPSubset(const CBioseq_set& bioseqSet)
{
    return (bioseqSet.IsSetClass() && 
            (bioseqSet.GetClass() == CBioseq_set::eClass_parts ||
            bioseqSet.GetClass() == CBioseq_set::eClass_segset));
 
}
 
 
static void s_ScanWhilePossibleNPSet(const CBioseq_set& bioseqSet,
        int& num_nucs, int& num_prots, bool& hasInvalidSubset)
{
    if (bioseqSet.IsSetSeq_set()) {
        for (const auto& pSubEntry : bioseqSet.GetSeq_set()) {
            if (pSubEntry) {
                if (pSubEntry->IsSeq()) {
                    s_IncrementSeqCount(pSubEntry->GetSeq(), num_nucs, num_prots);
                } else {
                    const auto& bioseqSet = pSubEntry->GetSet();
                    if (!s_IsValidNPSubset(bioseqSet)) {
                        hasInvalidSubset = true;
                        return;
                    }
                    s_ScanWhilePossibleNPSet(bioseqSet, num_nucs, num_prots, hasInvalidSubset);     
                }  
                if (num_nucs > 1) {
                    return;
                } 
            }
        }
    }
}
 
 
static bool s_LooksLikeNucProtSet(const CBioseq_set& bioseqSet)
{
    int numNucs{0};
    int numProts{0};
    bool hasInvalidSubset{false};
    s_ScanWhilePossibleNPSet(bioseqSet, numNucs, numProts, hasInvalidSubset);
 
    return (!hasInvalidSubset && numNucs == 1 && numProts > 0);
}
 
 
void CNewCleanup_imp::SeqsetBC(CBioseq_set& bss)
{
    if (bss.IsSetClass() &&
        (bss.GetClass() != CBioseq_set::eClass_not_set &&
         bss.GetClass() != CBioseq_set::eClass_other)) {
        return;
    }
 
    if( s_LooksLikeNucProtSet(bss) ) {
        bss.SetClass( CBioseq_set::eClass_nuc_prot );
        ChangeMade(CCleanupChange::eChangeBioseqSetClass);
    } else {
        bss.SetClass( CBioseq_set::eClass_genbank );
        ChangeMade(CCleanupChange::eChangeBioseqSetClass);
    }
}
 
static CMolInfo::TCompleteness GetCompletenessFromFlags(bool partial5, bool partial3, bool partial)
{
    CMolInfo::TCompleteness comp = CMolInfo::eCompleteness_complete;
    if (partial5 && partial3) {
        comp = CMolInfo::eCompleteness_no_ends;
    } else if (partial5) {
        comp = CMolInfo::eCompleteness_no_left;
    } else if (partial3) {
        comp = CMolInfo::eCompleteness_no_right;
    } else if (partial) {
        comp = CMolInfo::eCompleteness_partial;
    }
    return comp;
}
 
void CNewCleanup_imp::ProtSeqBC (CBioseq& bs)
{
    // Bail if not protein
    if (!bs.IsSetInst()) {
        return;
    }
    CSeq_inst& inst = bs.SetInst();
    if (!inst.IsSetMol() || inst.GetMol() != CSeq_inst::eMol_aa) {
        return;
    }
 
    if (bs.GetInst().IsSetTopology() && bs.GetInst().GetTopology() == CSeq_inst::eTopology_linear) {
        bs.SetInst().ResetTopology();
        ChangeMade(CCleanupChange::eChangeBioseqInst);
    }
 
    // Bail if no GIBBSQ ID
    if (!bs.IsSetId()) {
        return;
    }
    bool has_gibbsq = false;
    ITERATE(CBioseq::TId, id, bs.GetId()) {
       if ((*id)->IsGibbsq()) {
           has_gibbsq = true;
           break;
       }
    }
    if (!has_gibbsq) {
        return;
    }
 
    // Bail if no title or no partialness clues in title
    if (!bs.IsSetDescr()) {
        return;
    }
    bool make_partial5 = false;
    bool make_partial3 = false;
    for (auto dit : bs.GetDescr().Get()) {
        if (dit->IsTitle()) {
            if (NStr::Find(dit->GetTitle(), "{C-terminal}") != string::npos) {
                make_partial5 = true;
            }
            if (NStr::Find(dit->GetTitle(), "{N-terminal}") != string::npos) {
                make_partial3 = true;
            }
            break;
        }
    }
 
    if (!make_partial5 && !make_partial3) {
        return;
    }
 
    // Bail if no protein feature with missing partials
    if (!bs.IsSetAnnot()) {
        return;
    }
    for (auto ait : bs.SetAnnot()) {
        if (ait->IsSetData() && ait->GetData().IsFtable()) {
            for (auto fi : ait->SetData().SetFtable()) {
                if (fi->IsSetData() &&
                    fi->GetData().GetSubtype() == CSeqFeatData::eSubtype_prot &&
                    fi->IsSetPartial() && fi->GetPartial() &&
                    fi->IsSetLocation() &&
                    !fi->GetLocation().IsPartialStart(eExtreme_Biological) &&
                    !fi->GetLocation().IsPartialStop(eExtreme_Biological)) {
                    // note - we are only fixing partials if *both*
                    // ends were left as complete. One end being
                    // set as partial means that someone was doing this
                    // deliberately.
                    if (make_partial5) {
                        fi->SetLocation().SetPartialStart(true, eExtreme_Biological);
                    }
                    if (make_partial3) {
                        fi->SetLocation().SetPartialStop(true, eExtreme_Biological);
                    }
                    ChangeMade(CCleanupChange::eChangeSeqloc);
 
                    CMolInfo::TCompleteness wanted = GetCompletenessFromFlags(make_partial5, make_partial3, true);
                    for (auto ds : bs.SetDescr().Set()) {
                        if (ds->IsMolinfo() &&
                            (!ds->GetMolinfo().IsSetCompleteness() ||
                                ds->GetMolinfo().GetCompleteness() != wanted)) {
                            ds->SetMolinfo().SetCompleteness(wanted);
                            ChangeMade(CCleanupChange::eChangeMolInfo);
                            break;
                        }
                    }
                }
            }
        }
    }
}
 
 
void CNewCleanup_imp::SeqIdBC( CSeq_id &seq_id )
{
    // try to find CObject_id in Seq-id for certain types
    CRef<CObject_id> pObjectId;
    if( seq_id.IsLocal() ) {
        pObjectId.Reset( & GET_MUTABLE(seq_id, Local) );
    }
 
    // currently, we only process the Str ones
    if( ! pObjectId || ! FIELD_IS(*pObjectId, Str) ) {
        return;
    }
 
    x_TruncateSpacesMarkChanged( GET_MUTABLE(*pObjectId, Str) );
}
 
// change the target string by searching for the given search_pattern
// and replacing it with replacement up to max_replace times (0 means unlimited)
//
// Example:
//     string foo = "Test:   FOO   BAR    :BAZ."
//     s_RegexpReplace( foo, ":[ ]+", ": " );
// This turns foo into "Test: FOO   BAR    :BAZ."
// Returns "true" if a replacement was done
 
static const int s_RegexpReplace_UnlimitedReplacements = 0;
 
static
bool s_RegexpReplace( string &target,
    const char *search_pattern,
    const char *replacement,
    int max_replace = s_RegexpReplace_UnlimitedReplacements,
    CRegexp::ECompile compile_flags = CRegexp::fCompile_default )
{
    CRegexpUtil replacer( target );
    size_t num_replacements = replacer.Replace( search_pattern, replacement,
        compile_flags, CRegexp::fMatch_default, max_replace );
    // swap is faster than assignment
    replacer.GetResult().swap( target );
 
    return ( num_replacements > 0 );
}
 
// This is similar to lexicographical_compare_3way,
// but we have to implement it ourselves because
// it's an SGI extension, not in the standard.
template <class Iter1, class Iter2, class Compare>
static int ncbi_lexicographical_compare_3way(
    Iter1 first1, Iter1 last1,
    Iter2 first2, Iter2 last2,
    Compare compare )
{
    for( ; first1 != last1 && first2 != last2 ; ++first1, ++first2 ) {
        int comparison = compare( *first1, *first2 );
        if( comparison != 0 ) {
            return comparison;
        }
    }
 
    if( first1 == last1 ) {
        if( first2 == last2 ) {
            return 0; // they're equal
        } else {
            // second is longer
            return -1;
        }
    } else {
        // first is longer
        return 1;
    }
}
 
class PNocase_EqualChar
{
public:
    bool operator()( const char ch1, const char ch2 ) const {
        return toupper(ch1) == toupper(ch2);
    }
};
 
class PNocase_LessChar
{
public:
    bool operator()( const char ch1, const char ch2 ) const {
        return toupper(ch1) < toupper(ch2);
    }
};
 
class PNocase_CompareChar
{
public:
    int operator()( const char ch1, const char ch2 ) const {
        return ( (int)toupper(ch1) - (int)toupper(ch2) );
    }
};
 
// C compares using toupper, as opposed to the built-in
// stuff which seems to use tolower, thus producing
// some differences in sorting order in some places.
// Once we've fully moved away from C there's probably
// no harm in replacing all calls to s_CompareNoCaseCStyle with
// normal functions like NStr::CompareNocase()
static
int s_CompareNoCaseCStyle( const string &s1, const string &s2 )
{
    return ncbi_lexicographical_compare_3way(
            s1.begin(), s1.end(),
            s2.begin(), s2.end(),
            PNocase_CompareChar() );
}
 
static
const string &s_GenomeToPlastidName( const CBioSource& biosrc )
{
    SWITCH_ON_BIOSOURCE_GENOME (biosrc) {
    case NCBI_GENOME(apicoplast):
        {
            const static string apicoplast("apicoplast");
            return apicoplast;
        }
        break;
    case NCBI_GENOME(chloroplast):
        {
            const static string chloroplast("chloroplast");
            return chloroplast;
        }
        break;
    case NCBI_GENOME(chromoplast):
        {
            const static string chromoplast("chromoplast");
            return chromoplast;
        }
        break;
    case NCBI_GENOME(kinetoplast):
        {
            const static string kinetoplast("kinetoplast");
            return kinetoplast;
        }
        break;
    case NCBI_GENOME(leucoplast):
        {
            const static string leucoplast("leucoplast");
            return leucoplast;
        }
        break;
    case NCBI_GENOME(plastid):
        {
            const static string plastid("plastid");
            return plastid;
        }
        break;
    case NCBI_GENOME(proplastid):
        {
            const static string proplastid("proplastid");
            return proplastid;
        }
        break;
    default:
        return kEmptyStr;
        break;
    }
    return kEmptyStr;
}
 
// If str starts with prefix, the prefix is removed from the string.
static
bool s_RemoveInitial( string &str, const string &prefix, NStr::ECase case_to_use )
{
    if( NStr::StartsWith( str, prefix, case_to_use ) ) {
        str.erase( 0, prefix.length() );
        return true;
    }
    return false;
}
 
// Given the position of the opening paren in a string, this returns
// the position of the closing paren (keeping track of any nested parens
// in the middle.
// It returns NPOS if the paren is not closed.
// This function is not currently smart; it doesn't know about quotes
// or anything
static
SIZE_TYPE s_MatchingParenPos( const string &str, SIZE_TYPE open_paren_pos )
{
    _ASSERT( str[open_paren_pos] == '(' );
    _ASSERT( open_paren_pos < str.length() );
 
    // nesting level. start at 1 since we know there's an open paren
    int level = 1;
 
    SIZE_TYPE pos = open_paren_pos + 1;
    for( ; pos < str.length(); ++pos ) {
        switch( str[pos] ) {
            case '(':
                // nesting deeper
                ++level;
                break;
            case ')':
                // closed a level of nesting
                --level;
                if( 0 == level ) {
                    // reached the top: we're closing the initial paren,
                    // so we return our position
                    return pos;
                }
                break;
            default:
                // ignore other characters.
                // maybe in the future we'll handle ignoring parens in quotes or
                // things like that.
                break;
        }
    }
    return NPOS;
}
 
static bool s_AccessionCompare (
    const string& str1,
    const string& str2
)
 
{
    return ( NStr::CompareNocase( str1, str2 ) < 0 );
}
 
static bool s_AccessionEqual (
    const string& str1,
    const string& str2
)
 
{
    if (NStr::EqualNocase (str1, str2)) return true;
 
    return false;
}
 
 
void CNewCleanup_imp::GBblockOriginBC ( string& str )
{
    if (CleanVisStringJunk(str)) {
        ChangeMade (CCleanupChange::eTrimSpaces);
    }
}
 
 
void CNewCleanup_imp::GBblockBC (
    CGB_block& gbk
)
 
{
    CLEAN_STRING_LIST (gbk, Extra_accessions);
 
    if (! EXTRAACCN_ON_GENBANKBLOCK_IS_SORTED (gbk, s_AccessionCompare)) {
        SORT_EXTRAACCN_ON_GENBANKBLOCK (gbk, s_AccessionCompare);
        ChangeMade (CCleanupChange::eCleanQualifiers);
    }
 
    if (! EXTRAACCN_ON_GENBANKBLOCK_IS_UNIQUE (gbk, s_AccessionEqual)) {
        UNIQUE_EXTRAACCN_ON_GENBANKBLOCK (gbk, s_AccessionEqual);
        ChangeMade (CCleanupChange::eCleanQualifiers);
    }
 
    // split keywords at semicolons
    if (gbk.IsSetKeywords()) {
        string one_string = NStr::Join(gbk.GetKeywords(), ";");
        gbk.ResetKeywords();
        NStr::Split(one_string, ";", gbk.SetKeywords());
    }
 
    CLEAN_STRING_LIST (gbk, Keywords);
 
    CCachedRegexp reassembly_regex
        = regexpCache.Get("^tpa(?:_|[_:]re)assembly$",
                          CRegexp::fCompile_ignore_case);
    EDIT_EACH_KEYWORD_ON_EMBLBLOCK(keyword_it, gbk) {
        string & sKeyword = *keyword_it;
        if( reassembly_regex->IsMatch(sKeyword) ) {
            sKeyword = "TPA:assembly";
            ChangeMade (CCleanupChange::eCleanQualifiers);
        }
    }
 
    if( m_IsEmblOrDdbj ) {
        UNIQUE_WITHOUT_SORT_KEYWORD_ON_GENBANKBLOCK( gbk, PCase );
    } else {
        UNIQUE_WITHOUT_SORT_KEYWORD_ON_GENBANKBLOCK( gbk, PNocase );
    }
 
    CLEAN_STRING_MEMBER_JUNK (gbk, Source);
    if( FIELD_EQUALS(gbk, Source, ".") ) {
        RESET_FIELD(gbk, Source);
        ChangeMade(CCleanupChange::eRemoveQualifier);
    }
    if( FIELD_EQUALS(gbk, Origin, ".") ) {
        RESET_FIELD(gbk, Origin);
        ChangeMade(CCleanupChange::eRemoveQualifier);
    }
 
    CLEAN_STRING_MEMBER (gbk, Date);
    CLEAN_STRING_MEMBER (gbk, Div);
    CLEAN_STRING_MEMBER (gbk, Taxonomy);
}
 
void CNewCleanup_imp::EMBLblockBC (
    CEMBL_block& emb
)
 
{
    CLEAN_STRING_LIST (emb, Extra_acc);
 
    if (! EXTRAACCN_ON_EMBLBLOCK_IS_SORTED (emb, s_AccessionCompare)) {
        SORT_EXTRAACCN_ON_EMBLBLOCK (emb, s_AccessionCompare);
        ChangeMade (CCleanupChange::eCleanQualifiers);
    }
 
    if (! EXTRAACCN_ON_EMBLBLOCK_IS_UNIQUE (emb, s_AccessionEqual)) {
        UNIQUE_EXTRAACCN_ON_EMBLBLOCK (emb, s_AccessionEqual);
        ChangeMade (CCleanupChange::eCleanQualifiers);
    }
 
    CLEAN_STRING_LIST (emb, Keywords);
 
    UNIQUE_WITHOUT_SORT_KEYWORD_ON_EMBLBLOCK (emb, PCase);
}
 
 
// Give it a map that maps case-insensitive string to some other type,
// and it will return any matches that are a prefix for str.
// For example, if you have a mapping that includes ("foo" to 7), then passing
// str as "Foo something", will return the ("foo" to 7) mapping.
template< typename TMapType >
typename TMapType::const_iterator s_FindInMapAsPrefix( const string &str_arg, const TMapType &the_map )
{
    // holds the str we're looking at, which might be str_arg, or
    // might be another string constructed from it
    const string *str = &str_arg;
 
    // use this to delete strings created in this function, if any.
    // we don't read from it directly
    unique_ptr<string> temp_str;
 
    // chop off characters that can't be in the map, so they don't count
    SIZE_TYPE first_bad_char = 0;
    for( ; first_bad_char < str_arg.length(); ++first_bad_char ) {
        const char ch = str_arg[first_bad_char];
        if( ! isalnum(ch) && ch != '-' && ch != '_' && ch != ' ' ) {
            temp_str.reset( new string(str_arg, 0, first_bad_char) );
            str = temp_str.get();
            break;
        }
    }
 
    typename TMapType::const_iterator it = the_map.lower_bound( *str );
    if( it != the_map.begin() && ( it == the_map.end() || ! NStr::EqualNocase(*str, it->first) ) ) {
        --it;
    }
    if ( it != the_map.end() && NStr::StartsWith(*str, it->first, NStr::eNocase)) {
        return it;
    }
    return the_map.end();
}
 
// s_FindInMapAsPrefix, but for data structures like sets.
template< typename TSetType >
typename TSetType::const_iterator s_FindInSetAsPrefix( const string &str, const TSetType &the_set )
{
    typename TSetType::const_iterator it = the_set.lower_bound( str );
    if( it != the_set.begin() && ( it == the_set.end() || ! NStr::EqualNocase(str, *it) ) ) {
        --it;
    }
    if ( it != the_set.end() && NStr::StartsWith(str, *it, NStr::eNocase)) {
        return it;
    }
    return the_set.end();
}
 
 
// copy "str" because we're changing it anyway
// returns true if we found anything
static
bool s_StringHasOrgModPrefix(const string &str, string::size_type &out_val_start_pos, TORGMOD_SUBTYPE &out_subtype)
{
    SIZE_TYPE pos = str.find_first_of(": ="), pos2;
    if (pos != 0  &&  pos != NPOS
        &&  (pos2 = str.find_first_not_of(": =", pos)) != NPOS) {
        try {
            string val = str.substr(0, pos);
            COrgMod::TSubtype subtype = COrgMod::GetSubtypeValue(val, COrgMod::eVocabulary_insdc);
            if ( !COrgMod::IsDiscouraged(subtype) ) {
                out_subtype       = subtype;
                out_val_start_pos = pos2;
                return true;
            }
        } catch (CSerialException&) {
        }
    }
    return false;
}
 
// returns true if we found anything
static
bool s_StringHasSubSourcePrefix(const string &str, string::size_type &out_val_start_pos, TSUBSOURCE_SUBTYPE &out_subtype)
{
    SIZE_TYPE pos = str.find_first_of(": ="), pos2;
    if (pos != 0  &&  pos != NPOS
        &&  (pos2 = str.find_first_not_of(": =", pos)) != NPOS) {
        try {
            string val = str.substr(0, pos);
            CSubSource::TSubtype subtype;
            if (NStr::EqualNocase(val, "Lat-long") || NStr::EqualNocase(val, "Latitude-Longitude")) {
                subtype = CSubSource::eSubtype_lat_lon;
            } else {
                subtype = CSubSource::GetSubtypeValue(val, CSubSource::eVocabulary_insdc);
            }
            if ( subtype == CSubSource::eSubtype_fwd_primer_name ||
                 subtype == CSubSource::eSubtype_fwd_primer_seq ||
                 subtype == CSubSource::eSubtype_rev_primer_name ||
                 subtype == CSubSource::eSubtype_rev_primer_seq ||
                 !CSubSource::IsDiscouraged(subtype) ) {
                out_subtype       = subtype;
                out_val_start_pos = pos2;
                return true;
            }
        } catch (CSerialException&) {
        }
    } else {
        // did not find delimiters
        try {
            CSubSource::TSubtype subtype = CSubSource::GetSubtypeValue(str);
            if ( !CSubSource::IsDiscouraged(subtype) && CSubSource::NeedsNoText(subtype)) {
                out_subtype       = subtype;
                out_val_start_pos = str.length();
                return true;
            }
        } catch (CSerialException&) {
        }
    }
    return false;
}
 
 
// is st1 < st2
 
static bool s_SubsourceCompare (
    const CRef<CSubSource>& st1,
    const CRef<CSubSource>& st2
)
 
{
    const CSubSource& sbs1 = *(st1);
    const CSubSource& sbs2 = *(st2);
 
    TSUBSOURCE_SUBTYPE chs1 = GET_FIELD (sbs1, Subtype);
    TSUBSOURCE_SUBTYPE chs2 = GET_FIELD (sbs2, Subtype);
 
    if (chs1 < chs2) return true;
    if (chs1 > chs2) return false;
 
    if (FIELD_IS_SET (sbs2, Name)) {
        if (! FIELD_IS_SET (sbs1, Name)) return true;
        if (s_CompareNoCaseCStyle(GET_FIELD (sbs1, Name), GET_FIELD (sbs2, Name)) < 0) return true;
    }
 
    return false;
}
 
// Two SubSource's are equal and duplicates if:
// they have the same subtype
// and the same name (or don't require a name).
 
static bool s_SubsourceEqual (
    const CRef<CSubSource>& st1,
    const CRef<CSubSource>& st2
)
 
{
    const CSubSource& sbs1 = *(st1);
    const CSubSource& sbs2 = *(st2);
 
    TSUBSOURCE_SUBTYPE chs1 = GET_FIELD (sbs1, Subtype);
    TSUBSOURCE_SUBTYPE chs2 = GET_FIELD (sbs2, Subtype);
 
    if (chs1 != chs2) return false;
    if (CSubSource::NeedsNoText (chs2)) return true;
 
    if (FIELD_IS_SET (sbs1, Name) && FIELD_IS_SET (sbs2, Name)) {
        if (NStr::EqualNocase (GET_FIELD (sbs1, Name), GET_FIELD (sbs2, Name))) return true;
    }
    if (! FIELD_IS_SET (sbs1, Name) && ! FIELD_IS_SET (sbs2, Name)) return true;
 
    return false;
}
 
void CNewCleanup_imp::BiosourceFeatBC (
    CBioSource& biosrc,
    CSeq_feat & seqfeat
)
{
    // consolidate all orgmods of subtype "other" into one
    CRef<COrgMod> pFirstOtherOrgMod;
    if (biosrc.IsSetOrg() && biosrc.GetOrg().IsSetOrgname() && biosrc.GetOrg().GetOrgname().IsSetMod()) {
        auto& mod_set = biosrc.SetOrg().SetOrgname().SetMod();
        auto mod_it = mod_set.begin();
        while (mod_it != mod_set.end()) {
            COrgMod & orgmod = **mod_it;
 
            // we're only cleaning the ones of type "other"
            if (!FIELD_EQUALS(orgmod, Subtype, NCBI_ORGMOD(other)) ||
                !FIELD_IS_SET(orgmod, Subname))
            {
                ++mod_it;
                continue;
            }
 
            if (pFirstOtherOrgMod) {
                STRING_FIELD_APPEND(*pFirstOtherOrgMod, Subname, "; ", GET_STRING_FLD_OR_BLANK(orgmod, Subname));
                ChangeMade(CCleanupChange::eChangeOrgmod);
                mod_it = mod_set.erase(mod_it);
                ChangeMade(CCleanupChange::eRemoveOrgmod);
            } else {
                pFirstOtherOrgMod.Reset(&orgmod);
                ++mod_it;
            }
        }
    }
 
    // consolidate all subsources of subtype "other" into one
    CRef<CSubSource> pFirstOtherSubSource;
    EDIT_EACH_SUBSOURCE_ON_BIOSOURCE( subsrc_iter, biosrc ) {
        CSubSource &subsrc = **subsrc_iter;
 
        // we're only cleaning the ones of type "other"
        if( ! FIELD_EQUALS(subsrc, Subtype, NCBI_SUBSOURCE(other) ) ||
            ! FIELD_IS_SET(subsrc, Name) )
        {
            continue;
        }
 
        if( pFirstOtherSubSource ) {
            STRING_FIELD_APPEND(*pFirstOtherSubSource, Name, "; ", GET_STRING_FLD_OR_BLANK(subsrc, Name) );
            ChangeMade(CCleanupChange::eChangeSubsource);
            ERASE_SUBSOURCE_ON_BIOSOURCE(subsrc_iter, biosrc);
            ChangeMade(CCleanupChange::eRemoveSubSource);
        } else {
            pFirstOtherSubSource.Reset( &subsrc );
        }
    }
 
    // transfer feat comment (if any) to the end of the last other subsource note
    if( FIELD_IS_SET(seqfeat, Comment) ) {
 
        if( ! pFirstOtherSubSource ) {
            // create an empty subsource note if none found
            pFirstOtherSubSource.Reset( new CSubSource );
            SET_FIELD(*pFirstOtherSubSource, Subtype, NCBI_SUBSOURCE(other) );
            ADD_SUBSOURCE_TO_BIOSOURCE(biosrc, pFirstOtherSubSource);
        }
 
        STRING_FIELD_APPEND(*pFirstOtherSubSource, Name, "; ", GET_FIELD(seqfeat, Comment));
        ChangeMade ( CCleanupChange::eChangeSubsource );
        RESET_FIELD(seqfeat, Comment);
        ChangeMade ( CCleanupChange::eChangeComment );
    }
 
    // special orgmod cleanup just for features (yes, is stupid, but is what C toolkit does)
    if (biosrc.IsSetOrg() && biosrc.GetOrg().IsSetMod()) {
        EDIT_EACH_MOD_ON_ORGREF (it, biosrc.SetOrg()) {
            if (x_CompressSpaces(*it)) {
                ChangeMade ( CCleanupChange::eTrimSpaces );
            }
        }
    }
}
 
static void s_CorrectTildes (
    string& str
)
 
{
#ifndef NCBI_OS_MSWIN
    NStr::ReplaceInPlace (str, "were ~25 cm in height (~3 weeks)", "were ~~25 cm in height (~~3 weeks)");
    NStr::ReplaceInPlace (str, "generally ~3 weeks", "generally ~~3 weeks");
    NStr::ReplaceInPlace (str, "sequencing (~4 96-well plates)", "sequencing (~~4 96-well plates)");
    NStr::ReplaceInPlace (str, "size distribution (~2 kb)", "size distribution (~~2 kb)");
    NStr::ReplaceInPlace (str, "sequencing (~3 96-well plates)", "sequencing (~~3 96-well plates)");
    NStr::ReplaceInPlace (str, "vector. 1~2 ul of ligated", "vector. 1~~2 ul of ligated");
    /*
    NStr::ReplaceInPlace (str, "Lambda FLC I.~Islet cells were provided", "Lambda FLC I.~~Islet cells were provided");
    */
    NStr::ReplaceInPlace (str, "different strains~of mice", "different strains of mice");
    NStr::ReplaceInPlace (str, "oligo-dT-NotI primer~(5'-biotin", "oligo-dT-NotI primer (5'-biotin");
    NStr::ReplaceInPlace (str, "sizes of 200~800 bp were purified", "sizes of 200~~800 bp were purified");
    NStr::ReplaceInPlace (str, "Tween 20 (~50 ml per tree)", "Tween 20 (~~50 ml per tree)");
    NStr::ReplaceInPlace (str, "the SMART approach (~http://www.evrogen.com", "the SMART approach (http://www.evrogen.com");
    NStr::ReplaceInPlace (str, "the morning (~10 am) with", "the morning (~~10 am) with");
    NStr::ReplaceInPlace (str, "(host) sequences (~10%)", "(host) sequences (~~10%)");
    /*
    NStr::ReplaceInPlace (str, "unidirectionally.~ High quality", "unidirectionally. High quality");
    NStr::ReplaceInPlace (str, "onlysubmitted.~ Average", "onlysubmitted. Average");
    */
    NStr::ReplaceInPlace (str, "Plasmid; ~The F03-1270", "Plasmid; The F03-1270");
    NStr::ReplaceInPlace (str, "using STS-PCR~from Eb", "using STS-PCR from Eb");
    NStr::ReplaceInPlace (str, "specific to~the Eb", "specific to the Eb");
    NStr::ReplaceInPlace (str, "side of insert);  , M.F., Lennon", "side of insert); Bonaldo, M.F., Lennon");
    NStr::ReplaceInPlace (str, "Uni-ZAP XR vector. 1~2 ul of", "Uni-ZAP XR vector. 1~~2 ul of");
    NStr::ReplaceInPlace (str, "from diploid~Secale montanum", "from diploid Secale montanum");
    NStr::ReplaceInPlace (str, "homology with~U43516,", "homology with U43516,");
    /*
    NStr::ReplaceInPlace (str, "from http//www.biobase.dk/~ddbase", "from http//www.biobase.dk/~~ddbase");
    */
    NStr::ReplaceInPlace (str, "plasmid; ~Assembled EST", "plasmid; Assembled EST");
    NStr::ReplaceInPlace (str, "databases.~Different cDNA", "databases. Different cDNA");
    NStr::ReplaceInPlace (str, "enzyme PstI.~DH5-alpha", "enzyme PstI. DH5-alpha");
    NStr::ReplaceInPlace (str, "as they~were prepared", "as they were prepared");
    NStr::ReplaceInPlace (str, "loci in~the genome", "loci in the genome");
    NStr::ReplaceInPlace (str, "P{CaSpeR}Cp1~50C (FBti0004219)", "P{CaSpeR}Cp1~~50C (FBti0004219)");
    NStr::ReplaceInPlace (str, "seedlings with 2~4 leaves", "seedlings with 2~~4 leaves");
    NStr::ReplaceInPlace (str, "tween 20 (~50mLs per tree)", "tween 20 (~~50mLs per tree)");
#endif //NCBI_OS_MSWIN
}
 
 
bool s_SameSubtype(const CSubSource& s1, const CSubSource& s2)
{
    if (!s1.IsSetSubtype() && !s2.IsSetSubtype()) {
        return true;
    } else if (!s1.IsSetSubtype() || !s2.IsSetSubtype()) {
        return false;
    } else {
        return s1.GetSubtype() == s2.GetSubtype();
    }
}
 
 
// close enough if second name contains the first
bool s_NameCloseEnough(const CSubSource& s1, const CSubSource& s2)
{
    if (!s1.IsSetName() && !s2.IsSetName()) {
        return true;
    } else if (!s1.IsSetName() || !s2.IsSetName()) {
        return false;
    }
    const string& n1 = s1.GetName();
    const string& n2 = s2.GetName();
 
    if (NStr::Equal(n1, n2)) {
        return true;
    } else {
        return false;
    }
}
 
 
void CNewCleanup_imp::SubSourceListBC(CBioSource& biosrc)
{
    if (!biosrc.IsSetSubtype()) {
        return;
    }
 
    // sort and remove duplicates.
    if (biosrc.IsSetSubtype() && biosrc.GetSubtype().size() > 1) {
        if (!SUBSOURCE_ON_BIOSOURCE_IS_SORTED(biosrc, s_SubsourceCompare)) {
            SORT_SUBSOURCE_ON_BIOSOURCE(biosrc, s_SubsourceCompare);
            ChangeMade(CCleanupChange::eCleanSubsource);
        }
 
        // remove duplicates and subsources that contain previous values
        CBioSource::TSubtype::iterator s = biosrc.SetSubtype().begin();
        CBioSource::TSubtype::iterator s_next = s;
        ++s_next;
        while (s_next != biosrc.SetSubtype().end()) {
            if (s_SameSubtype(**s, **s_next) && s_NameCloseEnough(**s, **s_next)) {
                s = biosrc.SetSubtype().erase(s);
                ChangeMade(CCleanupChange::eCleanSubsource);
            } else {
                ++s;
            }
            ++s_next;
        }
    }
}
 
static string s_RepairISOCollDateTimeString (string& date_string)
{
    vector<string> components;
    NStr::Split(date_string, "T", components);
 
    if (components.size() == 1) {
        return date_string;
    }
 
    if (components.size() == 2) {
        string dat = components[0];
        string tim = components[1];
        size_t zee = tim.length();
        if (zee > 4 && tim[zee-1] == 'Z' && tim[1] == ':') {
            return dat + "T" + "0" + tim;
        }
    }
 
    return date_string;
}
 
static string s_RepairISOCollDateTimePair (string& coll_date)
{
    vector<string> pieces;
    NStr::Split(coll_date, "/", pieces);
 
    if (pieces.size() == 1) {
        string newdate = s_RepairISOCollDateTimeString(pieces[0]);
    }
 
    if (pieces.size() == 2) {
        string fstdate = s_RepairISOCollDateTimeString(pieces[0]);
        string scddate = s_RepairISOCollDateTimeString(pieces[1]);
        return fstdate + "/" + scddate;
    }
 
    return coll_date;
}
 
string s_CleanupLatLon( string &subname ) {
    string lat;
    string north_or_south;
    string lon;
    string east_or_west;
 
    if (subname.length() < 1) {
       return subname;
    }
    char ch = subname[0];
    if (ch < '0' || ch > '9') {
        return subname;
    }
 
    // extract the pieces
    CNcbiIstrstream lat_lon_stream( subname );
    lat_lon_stream >> lat;
    lat_lon_stream >> north_or_south;
    lat_lon_stream >> lon;
    lat_lon_stream >> east_or_west;
    if( lat_lon_stream.bad() ) {
        return subname;
    }
 
    if( north_or_south != "N" && north_or_south != "S" ) {
        return subname;
    }
 
    if( east_or_west != "E" && east_or_west != "W" ) {
        return subname;
    }
 
    size_t pos = NStr::Find(lat, ".");
    if (pos > 0) {
        size_t len = lat.length();
        if (pos + 9 < len) {
            lat.erase(pos + 9);
        }
    }
 
    pos = NStr::Find(lon, ".");
    if (pos > 0) {
        size_t len = lon.length();
        if (pos + 9 < len) {
            lon.erase(pos + 9);
        }
    }
 
    return lat + " " + north_or_south + " " + lon + " " + east_or_west;
}
 
void CNewCleanup_imp::BiosourceBC (
    CBioSource& biosrc
)
{
    if( FIELD_EQUALS( biosrc, Genome, CBioSource::eGenome_virion ) ) {
        RESET_FIELD( biosrc, Genome );
        ChangeMade ( CCleanupChange::eChangeBioSourceGenome );
    }
 
    if( FIELD_EQUALS( biosrc, Origin, NCBI_ORIGIN(unknown) ) ) {
        RESET_FIELD(biosrc, Origin);
        ChangeMade ( CCleanupChange::eChangeBioSourceOrigin );
    }
 
    // remove spaces and convert to lowercase in fwd_primer_seq and rev_primer_seq.
    if( FIELD_IS_SET(biosrc, Subtype) ) {
        SUBSOURCE_ON_BIOSOURCE_Type::iterator prev =
            SUBSOURCE_ON_BIOSOURCE_Set(biosrc).end();
        EDIT_EACH_SUBSOURCE_ON_BIOSOURCE (it, biosrc) {
            CSubSource& sbs = **it;
 
            TSUBSOURCE_SUBTYPE chs = GET_FIELD (sbs, Subtype);
            if (CSubSource::NeedsNoText (chs)) {
                // name is required - set it to empty string
                if( ! FIELD_IS_SET(sbs, Name) || ! GET_FIELD(sbs, Name).empty() ) {
                    SET_FIELD (sbs, Name, "");
                    ChangeMade(CCleanupChange::eCleanSubsource);
                }
                CLEAN_STRING_MEMBER(sbs, Attrib);
            } else {
                CLEAN_AND_COMPRESS_STRING_MEMBER(sbs, Name);
                if( ! FIELD_IS_SET(sbs, Name) ) {
                    // name must be set
                    SET_FIELD (sbs, Name, "");
                    ChangeMade(CCleanupChange::eCleanSubsource);
                }
                x_RemoveFlankingQuotes( GET_MUTABLE(sbs, Name) );
                CLEAN_STRING_MEMBER(sbs, Attrib);
            }
 
            if( chs == NCBI_SUBSOURCE(country) ) {
                string &country = GET_MUTABLE(sbs, Name);
                static const string kUSPrefix( "United States:" );
                if( NStr::EqualNocase(country, "United States") ||
                    NStr::EqualNocase(country, "United States of America") ||
                    NStr::EqualNocase(country, "U.S.A.") )
                {
                    country = "USA";
                    ChangeMade(CCleanupChange::eCleanSubsource);
                } else if( NStr::StartsWith(country, kUSPrefix, NStr::eNocase) ) {
                    country.replace( 0, kUSPrefix.length(), "USA:" );
                    ChangeMade(CCleanupChange::eCleanSubsource);
                }
 
            }
 
            if( chs == NCBI_SUBSOURCE(altitude) ) {
                string &altitude = GET_MUTABLE(sbs, Name);
 
                // normalize units part (that is, the ending) if possible
                // (e.g. "meters", etc. to "m.")
                // Note that we do NOT count a match if it's just a number because
                // we can't be sure that the submitter wasn't thinking "feet" or whatever.
                CCachedRegexp altitude_regex = regexpCache.Get(
                    "^([+-]?[0-9]+(\\.[0-9]+)?) ?(m|meter[s]?|metre[s]?)\\.?$",
                    CRegexp::fCompile_ignore_case );
 
                if( altitude_regex->IsMatch(altitude) ) {
                    string new_altitude = altitude_regex->GetSub(altitude, 1);
                    new_altitude += " m";
                    if( altitude != new_altitude ) {
                        altitude = new_altitude;
                        ChangeMade(CCleanupChange::eCleanSubsource);
                    }
                }
            }
 
            if( chs == NCBI_SUBSOURCE(lat_lon) ) {
                string &lat_lon = GET_MUTABLE(sbs, Name);
 
                string subname = s_CleanupLatLon(lat_lon);
                if ( lat_lon != subname ) {
                    lat_lon = subname;
                    ChangeMade(CCleanupChange::eCleanSubsource);
                }
            }
 
            /*
            if( chs == NCBI_SUBSOURCE(lat_lon) ) {
                string &lat_lon = GET_MUTABLE(sbs, Name);
 
                CCachedRegexp lat_lon_with_comma = regexpCache.Get(
                    "^[-.0-9]+ ., [-.0-9]+ .$");
                if( lat_lon_with_comma->IsMatch(lat_lon) ) {
                    // remove the comma
                    SIZE_TYPE comma_pos = lat_lon.find(',');
                    _ASSERT(comma_pos != NPOS );
                    lat_lon.erase(comma_pos, 1);
                    ChangeMade(CCleanupChange::eCleanSubsource);
                }
            }
            */
 
            if ( chs == NCBI_SUBSOURCE(collection_date) ) {
                string &coll_date = GET_MUTABLE(sbs, Name);
                string new_date = s_RepairISOCollDateTimePair(coll_date);
                if (!NStr::Equal(new_date, coll_date)) {
                    coll_date = new_date;
                    ChangeMade(CCleanupChange::eCleanSubsource);
                }
            }
 
           if ( chs == NCBI_SUBSOURCE(fwd_primer_seq) ||
                chs == NCBI_SUBSOURCE(rev_primer_seq) )
            {
                const string before = GET_FIELD (sbs, Name);
                CPCRPrimerSeq::Clean( GET_MUTABLE(sbs, Name) );
                const string& after = GET_FIELD (sbs, Name);
                if ( before != after ) {
                    ChangeMade (CCleanupChange::eCleanSubsource);
                }
            }
 
            // determine whether we should remove this subsource:
            if(  (! FIELD_IS_SET(sbs, Name) || GET_FIELD(sbs, Name).empty()) &&
                ! CSubSource::NeedsNoText( chs ) )
            {
                ERASE_SUBSOURCE_ON_BIOSOURCE(it, biosrc);
                ChangeMade(CCleanupChange::eCleanSubsource);
                continue;
            } else if( chs == NCBI_SUBSOURCE(plastid_name) &&
                STRING_FIELD_MATCH(sbs, Name, s_GenomeToPlastidName(biosrc) ) )
            {
                ERASE_SUBSOURCE_ON_BIOSOURCE(it, biosrc);
                ChangeMade(CCleanupChange::eCleanSubsource);
                continue;
            } else if( prev != SUBSOURCE_ON_BIOSOURCE_Set(biosrc).end() ) {
                TSUBSOURCE_SUBTYPE prev_chs = GET_FIELD (**prev, Subtype);
                const string &name = GET_FIELD(sbs, Name);
                const string &prev_name = GET_FIELD(**prev, Name);
 
                if ( (chs == prev_chs) &&
                    ( CSubSource::NeedsNoText(chs) ||
                    NStr::EqualNocase(prev_name, name) ||
                    (prev_chs == NCBI_SUBSOURCE(other) &&
                    NStr::Find(prev_name, name) != NPOS)))
                {
                    ERASE_SUBSOURCE_ON_BIOSOURCE(it, biosrc);
                    ChangeMade(CCleanupChange::eCleanSubsource);
                    continue;
                } else if ( (chs == prev_chs) &&
                    prev_chs == NCBI_SUBSOURCE(other) &&
                    NStr::Find (name, prev_name) != NPOS )
                {
                    (**prev).Assign( sbs );
                    ERASE_SUBSOURCE_ON_BIOSOURCE(it, biosrc);
                    ChangeMade(CCleanupChange::eCleanSubsource);
                    continue;
                }
            }
 
            prev = it;
        }
    }
 
    // sort and remove duplicates.
    SubSourceListBC(biosrc);
 
    // PCR Primers
    if( FIELD_IS_SET(biosrc, Pcr_primers) ) {
        PCRReactionSetBC( GET_MUTABLE(biosrc, Pcr_primers) );
        if( GET_FIELD(biosrc, Pcr_primers).Get().empty() ) {
            RESET_FIELD(biosrc, Pcr_primers);
            ChangeMade(CCleanupChange::eChangePCRPrimers);
        }
    }
 
    // correct specific cases of inconsistently applied tildes
    if (biosrc.IsSetOrg() && biosrc.GetOrg().IsSetOrgname()) {
        auto& orgname = biosrc.SetOrg().SetOrgname();
        if (orgname.IsSetMod()) {
            auto& mod_set = orgname.SetMod();
            for (auto& orgmod_it : mod_set) {
                COrgMod & orgmod = *orgmod_it;
 
                // we're only correcting tildes for the ones of type "other"
                if (!FIELD_EQUALS(orgmod, Subtype, NCBI_ORGMOD(other)) ||
                    !FIELD_IS_SET(orgmod, Subname))
                {
                    continue;
                }
 
                string &subname = GET_MUTABLE(orgmod, Subname);
                s_CorrectTildes(subname);
            }
        }
    }
 
    EDIT_EACH_SUBSOURCE_ON_BIOSOURCE( subsrc_iter, biosrc ) {
        CSubSource &subsrc = **subsrc_iter;
 
        // we're only correcting tildes for the ones of type "other"
        if( ! FIELD_EQUALS(subsrc, Subtype, NCBI_SUBSOURCE(other) ) ||
            ! FIELD_IS_SET(subsrc, Name) )
        {
            continue;
        }
 
        string &name = GET_MUTABLE(subsrc, Name);
        s_CorrectTildes(name);
    }
 
    if (biosrc.IsSetOrg()) {
        if (biosrc.GetOrg().IsSetOrgname()) {
            OrgnameBC(biosrc.SetOrg().SetOrgname(), biosrc.SetOrg());
        }
    }
 
    if (biosrc.FixEnvironmentalSample()) {
        ChangeMade(CCleanupChange::eChangeSubsource);
    }
    if (biosrc.RemoveNullTerms()) {
        ChangeMade(CCleanupChange::eChangeBioSourceOther);
    }
    if (biosrc.FixGenomeForQualifiers()) {
        ChangeMade(CCleanupChange::eChangeBioSourceGenome);
    }
 
    x_PostBiosource(biosrc);
    if (biosrc.IsSetOrg()) {
        x_PostOrgRef(biosrc.SetOrg());
    }
}
 
void CNewCleanup_imp::x_PostBiosource( CBioSource& biosrc )
{
    if( FIELD_EQUALS(biosrc, Genome, NCBI_GENOME(unknown) ) ) {
        RESET_FIELD(biosrc, Genome);
        ChangeMade(CCleanupChange::eChangeBioSourceGenome);
    }
 
    if (BIOSOURCE_HAS_SUBSOURCE (biosrc)) {
 
        // remove plastid-name subsource if the value is the same as the biosource location
        const string &plastid_name = s_GenomeToPlastidName( biosrc );
 
        bool plasmid_subsource_found = false;
        EDIT_EACH_SUBSOURCE_ON_BIOSOURCE (it, biosrc) {
            CSubSource& sbs = **it;
            TSUBSOURCE_SUBTYPE chs = GET_FIELD (sbs, Subtype);
            if (CSubSource::NeedsNoText (chs)) {
                if (sbs.IsSetName() && !NStr::IsBlank(sbs.GetName())) {
                    RESET_FIELD (sbs, Name);
                    SET_FIELD (sbs, Name, "");
                    ChangeMade (CCleanupChange::eCleanSubsource);
                }
            } else if (chs == NCBI_SUBSOURCE(plastid_name)) {
                // plasTid
                if (NStr::EqualNocase (GET_FIELD (sbs, Name), plastid_name)) {
                    ERASE_SUBSOURCE_ON_BIOSOURCE (it, biosrc);
                    ChangeMade (CCleanupChange::eCleanSubsource);
                }
            } else if ( chs == NCBI_SUBSOURCE(plasmid_name) ) {
                // plasMid
                plasmid_subsource_found = true;
            }
        }
 
        // set genome to "plasmid" under some conditions
        if( plasmid_subsource_found ) {
            if( ! FIELD_IS_SET(biosrc, Genome) ||
                GET_FIELD(biosrc, Genome) == NCBI_GENOME(unknown) ||
                GET_FIELD(biosrc, Genome) == NCBI_GENOME(genomic) )
            {
                biosrc.SetGenome( NCBI_GENOME(plasmid) );
                ChangeMade(CCleanupChange::eChangeBioSourceGenome);
            }
        }
 
        // remove those with no name unless it has a subtype that doesn't need a name.
        EDIT_EACH_SUBSOURCE_ON_BIOSOURCE (it, biosrc) {
            CSubSource& sbs = **it;
            if (FIELD_IS_SET (sbs, Name) && ! GET_FIELD(sbs, Name).empty() ) continue;
            TSUBSOURCE_SUBTYPE chs = GET_FIELD (sbs, Subtype);
            if (CSubSource::NeedsNoText (chs)) continue;
            ERASE_SUBSOURCE_ON_BIOSOURCE (it, biosrc);
            ChangeMade (CCleanupChange::eCleanSubsource);
        }
 
        // sort and remove duplicates.
        if (! SUBSOURCE_ON_BIOSOURCE_IS_SORTED (biosrc, s_SubsourceCompare)) {
            SORT_SUBSOURCE_ON_BIOSOURCE (biosrc, s_SubsourceCompare);
            ChangeMade (CCleanupChange::eCleanSubsource);
        }
 
        if (! SUBSOURCE_ON_BIOSOURCE_IS_UNIQUE (biosrc, s_SubsourceEqual)) {
            UNIQUE_SUBSOURCE_ON_BIOSOURCE (biosrc, s_SubsourceEqual);
            ChangeMade (CCleanupChange::eCleanSubsource);
        }
 
        REMOVE_IF_EMPTY_SUBSOURCE_ON_BIOSOURCE(biosrc);
    }
}
 
 
 
 
static bool s_DbtagIsBad (
    CDbtag& dbt
)
 
{
    if (! FIELD_IS_SET (dbt, Db)) return true;
    const string& db = GET_FIELD(dbt, Db);
    if (NStr::IsBlank (db)) return true;
    if( NStr::EqualNocase(db, "PID") ||
        NStr::EqualNocase(db, "PIDg") ||
        NStr::EqualNocase(db, "NID") ) {
            return true;
    }
 
    if (! FIELD_IS_SET( dbt, Tag)) return true;
    const CObject_id& oid = GET_FIELD(dbt, Tag);
 
    if (FIELD_IS (oid, Id)) {
        if (GET_FIELD (oid, Id) == 0) return true;
    } else if (FIELD_IS (oid, Str)) {
        const string& str = GET_FIELD (oid, Str);
        if (NStr::IsBlank (str)) return true;
    } else return true;
 
    return false;
}
 
void CNewCleanup_imp::OrgrefModBC (string& str)
{
    if (TrimSpacesSemicolonsAndCommas(str)) {
        ChangeMade (CCleanupChange::eTrimSpaces);
    }
}
 
void CNewCleanup_imp::OrgrefBC (COrg_ref& org)
 
{
    CLEAN_STRING_MEMBER (org, Taxname);
    CLEAN_STRING_MEMBER (org, Common);
    CLEAN_STRING_LIST (org, Syn);
 
    if (FIELD_IS_SET (org, Orgname)) {
        COrgName& onm = GET_MUTABLE (org, Orgname);
        OrgnameBC (onm, org);
    }
 
 
    if (ORGREF_HAS_DBXREF (org)) {
        vector< CRef< CDbtag > > new_dbtags;
        EDIT_EACH_DBXREF_ON_ORGREF (it, org) {
            CDbtag& dbt = **it;
            DbtagBC(dbt);
            x_SplitDbtag(dbt, new_dbtags );
        }
        if( ! new_dbtags.empty() ) {
            copy( new_dbtags.begin(), new_dbtags.end(), back_inserter( org.SetDb() ) );
            ChangeMade (CCleanupChange::eChangeDbxrefs);
        }
    }
}
 
void CNewCleanup_imp::x_PostOrgRef( COrg_ref& org )
{
    EDIT_EACH_DBXREF_ON_ORGREF (it, org) {
        CDbtag& dbt = **it;
        if (s_DbtagIsBad (dbt)) {
            ERASE_DBXREF_ON_ORGREF (it, org);
            ChangeMade (CCleanupChange::eCleanDbxrefs);
        }
    }
 
    // sort/unique db_xrefs
    if (! DBXREF_ON_ORGREF_IS_SORTED (org, s_DbtagCompare)) {
        SORT_DBXREF_ON_ORGREF (org, s_DbtagCompare);
        ChangeMade (CCleanupChange::eCleanDbxrefs);
    }
    if (! DBXREF_ON_ORGREF_IS_UNIQUE (org, s_DbtagEqual)) {
        UNIQUE_DBXREF_ON_ORGREF (org, s_DbtagEqual);
        ChangeMade (CCleanupChange::eCleanDbxrefs);
    }
 
    // sort/unique syns
    if (! SYN_ON_ORGREF_IS_SORTED (org, s_OrgrefSynCompare)) {
        SORT_SYN_ON_ORGREF (org, s_OrgrefSynCompare);
        ChangeMade (CCleanupChange::eCleanOrgref);
    }
    if (! SYN_ON_ORGREF_IS_UNIQUE (org, s_OrgrefSynEqual)) {
        UNIQUE_SYN_ON_ORGREF (org, s_OrgrefSynEqual);
        ChangeMade (CCleanupChange::eCleanOrgref);
    }
 
}
 
// is om1 < om2
// to sort subtypes together.
 
static bool s_OrgModCompare (
    const CRef<COrgMod>& om1,
    const CRef<COrgMod>& om2
)
 
{
    const COrgMod& omd1 = *(om1);
    const COrgMod& omd2 = *(om2);
 
    // subtype comparison
    TORGMOD_SUBTYPE subtype1 = GET_FIELD (omd1, Subtype);
    TORGMOD_SUBTYPE subtype2 = GET_FIELD (omd2, Subtype);
    if (subtype1 < subtype2) return true;
    if (subtype1 > subtype2) return false;
 
    // subname comparison
    const string& subname1 = GET_FIELD (omd1, Subname);
    const string& subname2 = GET_FIELD (omd2, Subname);
    const int subname_comparison = NStr::CompareNocase( subname1, subname2 );
    if( subname_comparison < 0 ) {
        return true;
    } else if( subname_comparison > 0 ) {
        return false;
    }
 
    // attrib comparison (realistically, we don't expect to fall back to this)
    const string& attrib1 = ( FIELD_IS_SET(omd1, Attrib) ? GET_FIELD (omd1, Attrib) : kEmptyStr );
    const string& attrib2 = ( FIELD_IS_SET(omd2, Attrib) ? GET_FIELD (omd2, Attrib) : kEmptyStr );
 
    const int attrib_comparison = NStr::CompareNocase( attrib1, attrib2 );
    if (attrib_comparison < 0) {
        return true;
    } else {
        return false;
    }
}
 
// Two OrgMod's are equal and duplicates if:
// they have the same subname and same subtype
 
static bool s_OrgModEqual (
    const CRef<COrgMod>& om1,
    const CRef<COrgMod>& om2
)
 
{
    const COrgMod& omd1 = *(om1);
    const COrgMod& omd2 = *(om2);
 
    const string& subname1 = GET_FIELD (omd1, Subname);
    const string& subname2 = GET_FIELD (omd2, Subname);
    if (! NStr::EqualNocase (subname1, subname2)) return false;
 
    const string& attrib1 = ( FIELD_IS_SET(omd1, Attrib) ? GET_FIELD (omd1, Attrib) : kEmptyStr );
    const string& attrib2 = ( FIELD_IS_SET(omd2, Attrib) ? GET_FIELD (omd2, Attrib) : kEmptyStr );
    if (! NStr::EqualNocase (attrib1, attrib2)) return false;
 
    TORGMOD_SUBTYPE chs1 = GET_FIELD (omd1, Subtype);
    TORGMOD_SUBTYPE chs2 = GET_FIELD (omd2, Subtype);
    if (chs1 == chs2) return true;
 
    return false;
}
 
void CNewCleanup_imp::OrgnameBC (
    COrgName& onm, COrg_ref &org_ref
)
 
{
    CLEAN_STRING_MEMBER (onm, Attrib);
    CLEAN_STRING_MEMBER (onm, Lineage);
    CLEAN_STRING_MEMBER_JUNK (onm, Div);
 
    EDIT_EACH_ORGMOD_ON_ORGNAME (it, onm) {
        COrgMod& omd = **it;
        OrgmodBC (omd);
        if (! FIELD_IS_SET (omd, Subname) || NStr::IsBlank (GET_FIELD (omd, Subname))) {
            ERASE_ORGMOD_ON_ORGNAME (it, onm);
            ChangeMade (CCleanupChange::eRemoveOrgmod);
        }
    }
 
    // erase structured notes that already match value
    // (Note: This is O(N^2).  Maybe worth converting to a faster algo?)
    EDIT_EACH_ORGMOD_ON_ORGNAME (it, onm) {
        COrgMod& omd = **it;
        if (omd.GetSubtype() == NCBI_ORGMOD(other)) {
            bool do_erase = false;
            string val_name, otherval;
            NStr::SplitInTwo( omd.GetSubname(), " =:", val_name, otherval );
            try {
                COrgMod::TSubtype subtype = COrgMod::GetSubtypeValue(val_name);
                NStr::TruncateSpacesInPlace(otherval);
                FOR_EACH_ORGMOD_ON_ORGNAME (match_it, onm) {
                    if ((*match_it)->GetSubtype() == subtype
                        && NStr::EqualCase((*match_it)->GetSubname(), otherval)) {
                        do_erase = true;
                        break;
                    }
                }
            } catch (CSerialException& ) {
            }
 
            if (do_erase) {
                ERASE_ORGMOD_ON_ORGNAME (it, onm);
                ChangeMade (CCleanupChange::eCleanOrgmod);
            }
        }
    }
 
    if (! ORGMOD_ON_ORGNAME_IS_SORTED (onm, s_OrgModCompare)) {
        SORT_ORGMOD_ON_ORGNAME (onm, s_OrgModCompare);
        ChangeMade (CCleanupChange::eCleanOrgmod);
    }
 
    // clean Orgmod list
    x_OrgnameModBC( onm, GET_STRING_FLD_OR_BLANK(org_ref, Common) );
 
    if (! ORGMOD_ON_ORGNAME_IS_SORTED (onm, s_OrgModCompare)) {
        SORT_ORGMOD_ON_ORGNAME (onm, s_OrgModCompare);
        ChangeMade (CCleanupChange::eCleanOrgmod);
    }
 
    if (! ORGMOD_ON_ORGNAME_IS_UNIQUE (onm, s_OrgModEqual)) {
        UNIQUE_ORGMOD_ON_ORGNAME (onm, s_OrgModEqual);
        ChangeMade (CCleanupChange::eCleanOrgmod);
    }
 
    REMOVE_IF_EMPTY_ORGMOD_ON_ORGNAME(onm);
}
 
static bool RemoveSpaceBeforeAndAfterColon (
    string& str
)
 
{
    // May need to create a custom implementation if this
    // regex becomes a bottleneck
    return s_RegexpReplace( str, "[ ]*:[ ]*", ":");
}
 
void CNewCleanup_imp::OrgmodBC (
    COrgMod& omd
)
{
    CLEAN_AND_COMPRESS_STRING_MEMBER (omd, Subname);
    if (FIELD_IS_SET (omd, Subname)) {
        x_TrimInternalSemicolonsMarkChanged( GET_MUTABLE(omd, Subname) );
        x_RemoveFlankingQuotes( GET_MUTABLE(omd, Subname) );
    }
 
    CLEAN_AND_COMPRESS_STRING_MEMBER (omd, Attrib);
 
    TORGMOD_SUBTYPE subtype = GET_FIELD (omd, Subtype);
 
    if( subtype == NCBI_ORGMOD(specimen_voucher) ||
        subtype == NCBI_ORGMOD(culture_collection) ||
        subtype == NCBI_ORGMOD(bio_material) )
    {
        if (FIELD_IS_SET (omd, Subname)) {
            string &subname = GET_MUTABLE (omd, Subname);
            const string::size_type old_len = subname.length();
            RemoveSpaceBeforeAndAfterColon (subname);
            NStr::ReplaceInPlace( subname, "::", ":", 0, 1 );
            if( old_len != subname.length() ) {
                ChangeMade (CCleanupChange::eTrimSpaces);
            }
        }
    }
 
    if (omd.RemoveAbbreviation()) {
        ChangeMade(CCleanupChange::eCleanOrgmod);
    }
}
 
bool s_IsAllDigits(const string& str)
{
    if (str.length() == 0) {
        return false;
    }
    bool all_digits = true;
    ITERATE(string, s, str) {
        if (!isdigit(*s)) {
            all_digits = false;
            break;
        }
    }
    return all_digits;
}
 
void CNewCleanup_imp::DbtagBC (
    CDbtag& dbtag
)
 
{
    if (! FIELD_IS_SET (dbtag, Db)) return;
    if (! FIELD_IS_SET (dbtag, Tag)) return;
 
    string& db = GET_MUTABLE (dbtag, Db);
    if (NStr::IsBlank (db)) return;
 
    size_t len = db.length();
    NStr::TruncateSpacesInPlace(db);
    if (len != db.length()) {
        ChangeMade(CCleanupChange::eTrimSpaces);
    }
 
    if (dbtag.GetTag().IsStr()) {
        if (TrimSpacesSemicolonsAndCommas(dbtag.SetTag().SetStr())) {
            ChangeMade(CCleanupChange::eTrimSpaces);
        }
    }
 
    if (NStr::EqualNocase(db, "Swiss-Prot")
        || NStr::EqualNocase (db, "SWISSPROT")
        || NStr::EqualNocase (db, "UniProt/Swiss-Prot")) {
        db = "UniProtKB/Swiss-Prot";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "SPTREMBL")  ||
               NStr::EqualNocase(db, "TrEMBL")  ||
               NStr::EqualNocase(db, "UniProt/TrEMBL") ) {
        db = "UniProtKB/TrEMBL";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "SUBTILIS")) {
        db = "SubtiList";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "LocusID")) {
        db = "GeneID";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "MaizeDB")) {
        db = "MaizeGDB";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "GeneW")) {
        db = "HGNC";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "MGD")) {
        db = "MGI";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "IFO")) {
        db = "NBRC";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase(db, "BHB") ||
        NStr::EqualNocase(db, "BioHealthBase")) {
        db = "IRD";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::Equal(db, "GENEDB")) {
        db = "GeneDB";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::Equal(db, "cdd")) {
        db = "CDD";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::Equal(db, "FlyBase")) {
        db = "FLYBASE";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::Equal(db, "GreengenesID")) {
        db = "Greengenes";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::Equal(db, "HMPID")) {
        db = "HMP";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::Equal(db, "ATCC (inhost)")) {
        db = "ATCC(in host)";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::Equal(db, "ATCC (dna)")) {
        db = "ATCC(dna)";
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    }
 
    CObject_id& oid = GET_MUTABLE (dbtag, Tag);
 
    if (FIELD_IS (oid, Id)) {
        const string& db = dbtag.GetDb();
        if (NStr::EqualNocase (db, "HGNC") || NStr::EqualNocase (db, "VGNC") || NStr::EqualNocase (db, "MGI") ) {
            int val = dbtag.GetTag().GetId();
            string str = db + ":" + NStr::IntToString(val);
            dbtag.SetTag().SetStr(str);
            ChangeMade(CCleanupChange::eChangeDbxrefs);
        }
    }
 
    if (! FIELD_IS (oid, Str)) return;
 
    string& str = GET_MUTABLE(oid, Str);
    if (NStr::IsBlank (str)) return;
 
    db = dbtag.GetDb();
    str = dbtag.GetTag().GetStr();
    if (NStr::EqualNocase(db, "HPRD") && NStr::StartsWith (str, "HPRD_")) {
        dbtag.SetTag().SetStr (str.substr (5));
        ChangeMade(CCleanupChange::eChangeDbxrefs);
    } else if (NStr::EqualNocase (db, "MGI") ) {
        if (!NStr::Equal(db, "MGI")) {
            dbtag.SetDb("MGI");
            ChangeMade(CCleanupChange::eChangeDbxrefs);
        }
        if(NStr::StartsWith (str, "MGI:")) {
            /*
            dbtag.SetTag().SetStr (dbtag.GetTag().GetStr().substr (4));
            ChangeMade(CCleanupChange::eChangeDbxrefs);
            */
        }
        else if (NStr::StartsWith(str, "MGD:")) {
            dbtag.SetTag().SetStr("MGI:" + dbtag.GetTag().GetStr().substr(4));
            ChangeMade(CCleanupChange::eChangeDbxrefs);
        } else if (NStr::StartsWith(str, "J:")) {
            if (s_IsAllDigits(str.substr(2))) {
                dbtag.SetTag().SetStr("MGI:");
            }
        } else {
            string newstr = "MGI:" + str;
            dbtag.SetTag().SetStr(newstr);
            ChangeMade(CCleanupChange::eChangeDbxrefs);
        }
    } else if (NStr::EqualNocase (db, "HGNC") ) {
        if(! NStr::StartsWith (str, "HGNC:")) {
            string newstr = "HGNC:" + str;
            dbtag.SetTag().SetStr(newstr);
            ChangeMade(CCleanupChange::eChangeDbxrefs);
            /*
            dbtag.SetTag().SetStr (dbtag.GetTag().GetStr().substr (5));
            ChangeMade(CCleanupChange::eChangeDbxrefs);
            */
        }
    } else if (NStr::EqualNocase (db, "VGNC") ) {
        if(! NStr::StartsWith (str, "VGNC:")) {
            string newstr = "VGNC:" + str;
            dbtag.SetTag().SetStr(newstr);
            ChangeMade(CCleanupChange::eChangeDbxrefs);
            /*
            dbtag.SetTag().SetStr (dbtag.GetTag().GetStr().substr (5));
            ChangeMade(CCleanupChange::eChangeDbxrefs);
            */
        }
    } else if (NStr::EqualNocase (db, "RGD") ) {
        if(NStr::StartsWith (str, "RGD:")) {
            dbtag.SetTag().SetStr (str.substr (4));
            ChangeMade(CCleanupChange::eChangeDbxrefs);
        }
    }
 
    /*
    // convert to number if all digits
    if (s_IsAllDigits(str) && !NStr::StartsWith(str, "0")) {
        try {
            // extract the part before the first space for conversion
            string::size_type pos_of_first_space = 0;
            while (pos_of_first_space < str.length() && !isspace(str[pos_of_first_space])) {
                ++pos_of_first_space;
            }
            CTempString sStrOfNum(str, 0, pos_of_first_space);
 
            // only convert str to int if it fits into the non-negative side
            // of an int.
            int value = NStr::StringToInt(sStrOfNum, NStr::fConvErr_NoThrow);
            if (value > 0) {
                dbtag.SetTag().SetId(NStr::StringToUInt(sStrOfNum));
                ChangeMade(CCleanupChange::eChangeDbxrefs);
            }
        } catch (CStringException&) {
            // just leave things as are
        }
    }
    */
}
 
 
void CNewCleanup_imp::PubdescBC (
    CPubdesc& pubdesc
)
{
    if (CCleanupPub::CleanPubdesc(pubdesc, m_StripSerial)) {
        ChangeMade(CCleanupChange::eChangePublication);
    }
    // need to construct m_PubToNewPubLabelMap separately
    if (pubdesc.IsSetPub()) {
        for (auto p : pubdesc.SetPub().Set()) {
            string new_label;
            p->GetLabel(&new_label, CPub::eContent, true);
            m_PubToNewPubLabelMap[p] = new_label;
        }
    }
}
 
 
typedef pair<string, CRef<CPub> >   TCit;
struct TSortCit {
    bool operator ()(const TCit& c1, const TCit& c2) const {
 
        // First, try to compare case-insensitively
        // (We compare as if it were all-caps to match C's behavior )
        const int label_compare_no_case =  s_CompareNoCaseCStyle(c1.first, c2.first);
        if( label_compare_no_case != 0 ) {
            return (label_compare_no_case < 0);
        }
 
        // if they're the same, try to compare case-sensitively
        const int label_compare_case = NStr::CompareCase( c1.first, c2.first );
        if( label_compare_case != 0 ) {
            return (label_compare_case < 0);
        }
 
        // if they're still the same, fall back on cit-gen titles, if possible
        return CitGenTitlesLess(*c1.second, *c2.second);
    }
    bool CitGenTitlesLess(const CPub& p1, const CPub& p2) const {
        if ( ! p1.IsGen()  || ! p2.IsGen() ) {
            return false;
        }
        const CCit_gen& g1 = p1.GetGen();
        const CCit_gen& g2 = p2.GetGen();
        if ( g1.IsSetTitle() != g2.IsSetTitle() ) {
            return (g1.IsSetTitle() || g2.IsSetTitle());
        } else if( ! g1.IsSetTitle() && ! g2.IsSetTitle() ) {
            return false;
        }
        return g1.GetTitle() < g2.GetTitle();
    }
};
 
static
bool cmpSortedvsOld(const TCit& e1, const CRef<CPub>& e2) {
    return e1.second == e2;
}
 
void CNewCleanup_imp::PubSetBC( CPub_set &pub_set )
{
    // The Pub-set should always be pub. Ignore if not.
    if( ! pub_set.IsPub() ) {
        return;
    }
 
    // sort and unique by putting everything into a set
    // indexed by a label generated for each CPub.
    typedef set<TCit, TSortCit> TCitSet;
    TCitSet cit_set;
    for (auto cit_it : pub_set.GetPub()) {
        string label;
        cit_it->GetLabel(&label, CPub::eContent, CPub::fLabel_Unique, CPub::eLabel_V1 );
        // the following line may fail due to dups
        // (that's okay; it lets us automatically remove dups)
        cit_set.insert( TCit(label, cit_it) );
    }
    auto& publist = pub_set.SetPub();
    // Has anything been deleted, or has the order changed?
    if ( cit_set.size() != publist.size() ||
        ! equal(cit_set.begin(), cit_set.end(), publist.begin(), cmpSortedvsOld) )
    {
        // put everything left back into the feature's citation list.
        publist.clear();
        ITERATE (TCitSet, citset_it, cit_set) {
            publist.push_back(citset_it->second);
        }
        ChangeMade(CCleanupChange::eCleanCitonFeat);
    }
}
 
 
void CNewCleanup_imp::ImpFeatBC( CSeq_feat& feat )
{
    if( ! FIELD_IS_SET_AND_IS(feat, Data, Imp) ) {
        return;
    }
 
    CImp_feat &imf = GET_MUTABLE( feat.SetData(), Imp );
 
    CLEAN_STRING_MEMBER_JUNK(imf, Key);
    CLEAN_STRING_MEMBER(imf, Loc);
    CLEAN_STRING_MEMBER(imf, Descr);
 
    if (imf.IsSetKey() && CSeqFeatData::FixImportKey(imf.SetKey())) {
        ChangeMade(CCleanupChange::eChangeKeywords);
    }
 
    if ( FIELD_IS_SET(imf, Key) ) {
        const CImp_feat::TKey& key = GET_FIELD(imf, Key);
        if ( key == "satellite" && ! m_IsEmblOrDdbj ) {
            SET_FIELD(imf, Key, "repeat_region");
            ChangeMade(CCleanupChange::eChangeKeywords);
 
            CRef<CGb_qual> satellite_qual( new CGb_qual );
            satellite_qual->SetQual("satellite");
            string val;
            if( FIELD_IS_SET(feat, Comment) ) {
                val = x_ExtractSatelliteFromComment( GET_MUTABLE(feat, Comment) );
            }
            if( val.empty() ) {
                val = "satellite";
            }
            satellite_qual->SetVal( val );
 
            feat.SetQual().push_back( satellite_qual );
        } else if ( key == "LTR" ) {
            SET_FIELD(imf, Key, "repeat_region");
            ChangeMade(CCleanupChange::eChangeKeywords);
 
            CRef<CGb_qual> rpt_type_qual( new CGb_qual );
            rpt_type_qual->SetQual( "rpt_type" );
            rpt_type_qual->SetVal( "long_terminal_repeat" );
 
            feat.SetQual().push_back( rpt_type_qual );
        }
 
        CSeqFeatData::ESubtype subtype = feat.GetData().GetSubtype();
        if (CSeqFeatData::IsRegulatory(subtype)) {
            string regulatory_class = CSeqFeatData::GetRegulatoryClass(subtype);
            SET_FIELD(imf, Key, "regulatory");
            ChangeMade(CCleanupChange::eChangeKeywords);
            CRef<CGb_qual> regulatory_class_qual( new CGb_qual );
            regulatory_class_qual->SetQual("regulatory_class");
            if (NStr::IsBlank(regulatory_class)) {
                regulatory_class_qual->SetVal( "other" );
            } else {
                regulatory_class_qual->SetVal( regulatory_class );
            }
            feat.SetQual().push_back( regulatory_class_qual );
        }
 
        if( key == "repeat_region" && ! m_IsEmblOrDdbj ) {
            string val;
            if( FIELD_IS_SET(feat, Comment) ) {
                val = x_ExtractSatelliteFromComment( GET_MUTABLE(feat, Comment) );
            }
            if( ! val.empty() ) {
                CRef<CGb_qual> satellite_qual( new CGb_qual );
                satellite_qual->SetQual("satellite");
                satellite_qual->SetVal( val );
 
                feat.SetQual().push_back( satellite_qual );
                ChangeMade(CCleanupChange::eChangeKeywords);
            }
        }
 
        if( key == "CDS" ) {
            if( ! m_IsEmblOrDdbj ) {
                CRef<CCdregion> new_cdregion( new CCdregion );
                // get frame from location
                if( ! FIELD_EQUALS( feat, Pseudo, true ) &&
                    feat.IsSetLocation() &&
                    CCleanup::SetFrameFromLoc(*new_cdregion, feat.GetLocation(), *m_Scope)) {
                    ChangeMade(CCleanupChange::eChangeCdregion);
                }
                ChangeMade(CCleanupChange::eChangeKeywords);
 
                CdregionFeatBC( *new_cdregion, feat );
                feat.SetData().SetCdregion(*new_cdregion);
                return;
            }
        }
    }
 
    if( FIELD_IS_SET(imf, Loc) ) {
        if ( NStr::Find(imf.GetLoc(), "replace") != NPOS ) {
            x_AddReplaceQual(feat, imf.GetLoc());
            RESET_FIELD(imf, Loc);
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    } else if( FIELD_IS_SET(imf, Key) ) {
        const string &key = GET_FIELD(imf, Key);
 
        TRNAREF_TYPE rna_ref_type = NCBI_RNAREF(unknown);
        if ( key == "precursor_RNA" ) {
            rna_ref_type = NCBI_RNAREF(premsg);
        } else if ( key == "mRNA" ) {
            rna_ref_type = NCBI_RNAREF(mRNA);
        } else if ( key == "tRNA" ) {
            rna_ref_type = NCBI_RNAREF(tRNA);
        } else if ( key == "rRNA" ) {
            rna_ref_type = NCBI_RNAREF(rRNA);
        } else if ( key == "snRNA" ) {
            rna_ref_type = NCBI_RNAREF(snRNA);
        } else if ( key == "scRNA" ) {
            rna_ref_type = NCBI_RNAREF(scRNA);
        } else if ( key == "snoRNA" ) {
            rna_ref_type = NCBI_RNAREF(snoRNA);
        } else if ( key == "misc_RNA" ) {
            rna_ref_type = NCBI_RNAREF(other);
        }
        if (rna_ref_type != NCBI_RNAREF(unknown) ) {
            CRef<CRNA_ref> new_rna_ref( new CRNA_ref );
            new_rna_ref->SetType( rna_ref_type );
            feat.SetData().SetRna( *new_rna_ref );
            ChangeMade(CCleanupChange::eAddRNAref);
            x_CleanSeqFeatQuals(feat);
            RnaFeatBC(feat.SetData().SetRna(), feat);
        } else {
            TPROTREF_PROCESSED processed = NCBI_PROTREF(not_set);
            if ( key == "proprotein" ||  key == "preprotein" ) {
                processed = NCBI_PROTREF(preprotein);
            } else if ( key == "mat_peptide" ) {
                processed = NCBI_PROTREF(mature);
            } else if ( key == "sig_peptide" ) {
                processed = NCBI_PROTREF(signal_peptide);
            } else if ( key == "transit_peptide" ) {
                processed = NCBI_PROTREF(transit_peptide);
            } else if ( key == "propeptide" ) {
                processed = NCBI_PROTREF(propeptide);
            }
            if (processed != NCBI_PROTREF(not_set) || key == "Protein" ) {
                const CSeq_id* location_seq_id = ( feat.IsSetLocation() ? feat.GetLocation().GetId() : nullptr );
                if( location_seq_id ) {
                    CBioseq_Handle bioseq_handle = m_Scope->GetBioseqHandle(*location_seq_id);
                    if ( bioseq_handle && bioseq_handle.IsAa() ) {
                        CRef<CProt_ref> new_prot_ref( new CProt_ref );
                        new_prot_ref->SetProcessed( processed );
                        if (feat.IsSetComment() && !NStr::IsBlank(feat.GetComment())) {
                            new_prot_ref->SetName().push_back(feat.GetComment());
                            feat.ResetComment();
                        }
                        feat.SetData().SetProt( *new_prot_ref );
                        ChangeMade(CCleanupChange::eAddProtFeat);
                        x_CleanSeqFeatQuals(feat);
                    }
                }
            }
        }
    }
}
 
 
typedef SStaticPair<const char*, CSeqFeatData::TSite>  TSiteElem;
static const TSiteElem sc_site_map[] = {
    { "acetylation", CSeqFeatData::eSite_acetylation },
    { "active", CSeqFeatData::eSite_active },
    { "amidation", CSeqFeatData::eSite_amidation },
    { "binding", CSeqFeatData::eSite_binding },
    { "blocked", CSeqFeatData::eSite_blocked },
    { "cleavage", CSeqFeatData::eSite_cleavage },
    { "dna binding", CSeqFeatData::eSite_dna_binding },
    { "dna-binding", CSeqFeatData::eSite_dna_binding },
    { "gamma carboxyglutamic acid", CSeqFeatData::eSite_gamma_carboxyglutamic_acid },
    { "gamma-carboxyglutamic-acid", CSeqFeatData::eSite_gamma_carboxyglutamic_acid },
    { "glycosylation", CSeqFeatData::eSite_glycosylation },
    { "hydroxylation", CSeqFeatData::eSite_hydroxylation },
    { "inhibit", CSeqFeatData::eSite_inhibit },
    { "lipid binding", CSeqFeatData::eSite_lipid_binding },
    { "lipid-binding", CSeqFeatData::eSite_lipid_binding },
    { "metal binding", CSeqFeatData::eSite_metal_binding },
    { "metal-binding", CSeqFeatData::eSite_metal_binding },
    { "methylation", CSeqFeatData::eSite_methylation },
    { "modifi", CSeqFeatData::eSite_modified },
    { "mutagenized", CSeqFeatData::eSite_mutagenized },
    { "myristoylation", CSeqFeatData::eSite_myristoylation },
    { "nitrosylation", CSeqFeatData::eSite_nitrosylation },
    { "np binding", CSeqFeatData::eSite_np_binding },
    { "np-binding", CSeqFeatData::eSite_np_binding },
    { "oxidative deamination", CSeqFeatData::eSite_oxidative_deamination },
    { "oxidative-deamination", CSeqFeatData::eSite_oxidative_deamination },
    { "phosphorylation", CSeqFeatData::eSite_phosphorylation },
    { "pyrrolidone carboxylic acid", CSeqFeatData::eSite_pyrrolidone_carboxylic_acid },
    { "pyrrolidone-carboxylic-acid", CSeqFeatData::eSite_pyrrolidone_carboxylic_acid },
    { "signal peptide", CSeqFeatData::eSite_signal_peptide },
    { "signal-peptide", CSeqFeatData::eSite_signal_peptide },
    { "sulfatation", CSeqFeatData::eSite_sulfatation },
    { "transit peptide", CSeqFeatData::eSite_transit_peptide },
    { "transit-peptide", CSeqFeatData::eSite_transit_peptide },
    { "transmembrane region", CSeqFeatData::eSite_transmembrane_region },
    { "transmembrane-region", CSeqFeatData::eSite_transmembrane_region }
};
typedef CStaticArrayMap<string, CSeqFeatData::TSite, PNocase> TSiteMap;
DEFINE_STATIC_ARRAY_MAP_WITH_COPY(TSiteMap, sc_SiteMap, sc_site_map);
 
void CNewCleanup_imp::SiteFeatBC( const CSeqFeatData::ESite &site, CSeq_feat& feat )
{
    // If site set to "other", try to extract it from the comment
    if ( FIELD_IS_SET(feat, Comment)  &&
        (site == CSeqFeatData::TSite(0)  ||  site == CSeqFeatData::eSite_other))
    {
        // extract if comment starts with any informative possibilities listed in sc_SiteMap
        const string& comment = GET_FIELD(feat, Comment);
        TSiteMap::const_iterator it = s_FindInMapAsPrefix<TSiteMap>( comment, sc_SiteMap );
        if ( it != sc_SiteMap.end() ) {
            feat.SetData().SetSite(it->second);
            ChangeMade(CCleanupChange::eChangeSite);
            // erase the comment if it contains no further useful info aside from the site
            if (NStr::IsBlank(comment, it->first.length())  ||
                NStr::EqualNocase(comment, it->first.length(), NPOS, " site")) {
                    feat.ResetComment();
                    ChangeMade(CCleanupChange::eChangeComment);
            }
        }
    }
}
 
void CNewCleanup_imp::SeqLocBC( CSeq_loc &loc )
{
    switch (loc.Which()) {
    case CSeq_loc::e_Int :
        x_SeqIntervalBC( GET_MUTABLE(loc, Int) );
        break;
    case CSeq_loc::e_Packed_int :
        {
            CSeq_loc::TPacked_int::Tdata& ints = loc.SetPacked_int().Set();
            NON_CONST_ITERATE(CSeq_loc::TPacked_int::Tdata, interval_it, ints) {
                x_SeqIntervalBC(**interval_it);
            }
            if (ints.size() == 1) {
                CRef<CSeq_interval> int_ref = ints.front();
                loc.SetInt(*int_ref);
                ChangeMade(CCleanupChange::eChangeSeqloc);
            }
        }
        break;
    case CSeq_loc::e_Pnt :
        {
            CSeq_loc::TPnt& pnt = loc.SetPnt();
 
            if (pnt.IsSetStrand()) {
                if (pnt.GetStrand() == eNa_strand_unknown) {
                    pnt.SetStrand(eNa_strand_plus);
                    ChangeMade(CCleanupChange::eChangeStrand);
                }
            }
            else {
                pnt.SetStrand(eNa_strand_plus);
                ChangeMade(CCleanupChange::eChangeStrand);
            }
 
            // normalize Seq-point fuzz tl to tr and decrement position
            if (pnt.IsSetFuzz() && pnt.GetFuzz().IsLim() &&
                pnt.GetFuzz().GetLim() == CInt_fuzz::eLim_tl) {
                TSeqPos pos = pnt.GetPoint();
                if (pos > 0) {
                    pnt.SetFuzz().SetLim(CInt_fuzz::eLim_tr);
                    pnt.SetPoint(pos - 1);
                    ChangeMade(CCleanupChange::eChangeSeqloc);
                }
            }
        }
        break;
    case CSeq_loc::e_Mix :
        {
            typedef CSeq_loc::TMix::Tdata TMixList;
            // delete Null type Seq-locs from beginning and end of Mix list.
 
            // deleting from beginning:
            TMixList& sl_list = loc.SetMix().Set();
            TMixList::iterator sl_it = sl_list.begin();
            while (sl_it != sl_list.end()) {
                if ((*sl_it)->IsNull()) {
                    sl_it = sl_list.erase(sl_it);
                    ChangeMade(CCleanupChange::eChangeSeqloc);
                } else {
                    break;
                }
            }
 
            // deleting from end:
            if( sl_list.size() > 0 ) {
                sl_it = sl_list.end();
                while (sl_it != sl_list.begin()) {
                    --sl_it;
                    if ( ! (*sl_it)->IsNull()) {
                        break;
                    }
                }
                ++sl_it;
                if (sl_it != sl_list.end()) {
                    sl_list.erase(sl_it, sl_list.end());
                    ChangeMade(CCleanupChange::eChangeSeqloc);
                }
            }
 
            if (sl_list.size() == 0) {
                loc.SetNull();
                ChangeMade(CCleanupChange::eChangeSeqloc);
            } else if (sl_list.size() == 1) {
                CRef<CSeq_loc> only_sl = sl_list.front();
                loc.Assign(*only_sl);
                ChangeMade(CCleanupChange::eChangeSeqloc);
            }
        }
        break;
    default:
        break;
    }
 
    // don't allow strandedness on protein sequences
    {
        CBioseq_Handle bsh;
        if (m_Scope) {
            ITERATE( CSeq_loc, loc_ci, loc ) {
                bsh = m_Scope->GetBioseqHandle(loc_ci.GetSeq_id());
                if( bsh ) {
                    break;
                }
            }
        }
        if ( bsh && bsh.IsProtein() && FIELD_IS_SET(loc, Strand) ) {
            RESET_FIELD(loc, Strand);
            ChangeMade(CCleanupChange::eChangeStrand);
        }
    }
 
}
 
void CNewCleanup_imp::ConvertSeqLocWholeToInt( CSeq_loc &loc )
{
    if (loc.IsWhole()  &&  m_Scope) {
 
        // change the Seq-loc/whole to a Seq-loc/interval which covers the whole sequence.
        CRef<CSeq_id> id(new CSeq_id());
        id->Assign(loc.GetWhole());
        CBioseq_Handle bsh;
 
        if( id ) {
            bsh = m_Scope->GetBioseqHandle(*id);
        }
        if (bsh) {
            TSeqPos bs_len = bsh.GetBioseqLength();
            auto& interval = loc.SetInt();
            interval.SetId(*id);
            interval.SetFrom(0);
            interval.SetTo(bs_len - 1);
            ChangeMade(CCleanupChange::eChangeWholeLocation);
        }
    }
}
 
static void
s_AddSeqLocMix( CSeq_loc_mix::Tdata & new_mix_pieces,
               CSeq_loc_mix::Tdata & mix_pieces,
               bool any_nulls_seen )
{
    NON_CONST_ITERATE( CSeq_loc_mix::Tdata, old_mix_iter, mix_pieces ) {
        CRef<CSeq_loc> old_piece( *old_mix_iter );
        if( old_piece->IsNull() ) {
            // ignore
        } else if( old_piece->IsMix() ) {
            s_AddSeqLocMix( new_mix_pieces, old_piece->SetMix(),
                any_nulls_seen );
        } else {
            if( any_nulls_seen && ! new_mix_pieces.empty() ) {
                CRef<CSeq_loc> null_piece( new CSeq_loc );
                null_piece->SetNull();
                new_mix_pieces.push_back( null_piece );
            }
            new_mix_pieces.push_back( old_piece );
        }
    }
}
 
void CNewCleanup_imp::SeqLocMixBC( CSeq_loc_mix & loc_mix )
{
    if( ! loc_mix.IsSet() || loc_mix.Set().empty() ) {
        return;
    }
 
    // This function does two things simultaneously:
    // It checks for mix-inside-mix and also checks if
    // we need to do "NULL-normalization"
    bool have_seen_inner_mix = false;
    bool any_nulls_seen = false;
    bool alternates_not_null_then_null = true;
 
    CSeq_loc_mix::Tdata & mix_pieces = loc_mix.Set();
    if( (mix_pieces.size() % 2) == 0 ) {
        // can't do notnull-null-notnull-null-notnull-....-null-notnull
        // if we have an even number of items
        alternates_not_null_then_null = false;
    }
 
    bool last_piece_was_null = true;
    ITERATE( CSeq_loc_mix::Tdata, outer_mix_iter, mix_pieces ) {
        const CSeq_loc &this_piece = **outer_mix_iter;
        const bool this_piece_is_null = this_piece.IsNull();
 
        // see if we've found any NULLs in this loc
        if( this_piece_is_null ) {
            any_nulls_seen = true;
        }
 
        // see if we break alternation of notnull and null
        if( alternates_not_null_then_null ) {
            if( this_piece_is_null == last_piece_was_null ) {
                // two of the same kind in a row: does not alternate
                alternates_not_null_then_null = false;
            }
        }
 
        // see if there's a nested mix in here
        if( this_piece.IsMix() ) {
            have_seen_inner_mix = true;
            alternates_not_null_then_null = false; // mix breaks alternation
            // We have to check if the inner-mix contains any NULLs
            if( ! any_nulls_seen ) {
                CSeq_loc_CI inner_ci( this_piece, CSeq_loc_CI::eEmpty_Allow );
                for( ; inner_ci; ++inner_ci ) {
                    if( inner_ci.IsEmpty() ) {
                        any_nulls_seen = true;
                    }
                }
            }
        }
 
        // for next iteration
        last_piece_was_null = this_piece_is_null;
    }
 
    // we've examined the location, so if there are any problems, we have
    // to rebuild it.
    if( have_seen_inner_mix ||
        (any_nulls_seen && ! alternates_not_null_then_null) )
    {
        CSeq_loc_mix new_mix;
        CSeq_loc_mix::Tdata & new_mix_pieces = new_mix.Set();
 
        // has to be in a separate function because it's recursive
        s_AddSeqLocMix( new_mix_pieces, mix_pieces, any_nulls_seen );
 
        // swap is faster than assignment
        loc_mix.Set().swap( new_mix_pieces );
    }
}
 
static bool s_IsJustQuotes (const string& str)
 
{
    FOR_EACH_CHAR_IN_STRING (str_itr, str) {
        const char& ch = *str_itr;
        if (ch > ' ' && ch != '"' && ch != '\'') return false;
    }
    return true;
}
 
void CNewCleanup_imp::GBQualBC (
    CGb_qual& gbq
)
 
{
    CLEAN_STRING_MEMBER (gbq, Qual);
    if (! FIELD_IS_SET (gbq, Qual)) {
        SET_FIELD (gbq, Qual, kEmptyStr);
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
 
    if (FIELD_IS_SET (gbq, Val)) {
        const string::size_type old_length = gbq.GetVal().length();
        CleanVisString (gbq.SetVal());
        TrimInternalSemicolons (gbq.SetVal());
        x_CompressSpaces( gbq.SetVal() );
        if (gbq.GetVal().length() != old_length) {
            ChangeMade (CCleanupChange::eTrimSpaces);
        }
    }
    if (FIELD_IS_SET (gbq, Val) && s_IsJustQuotes (GET_FIELD (gbq, Val))) {
        SET_FIELD (gbq, Val, kEmptyStr);
        ChangeMade (CCleanupChange::eCleanDoubleQuotes);
    }
    if (! FIELD_IS_SET (gbq, Val)) {
        SET_FIELD (gbq, Val, kEmptyStr);
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
 
    _ASSERT (FIELD_IS_SET (gbq, Qual) && FIELD_IS_SET (gbq, Val));
 
    if (NStr::EqualNocase(gbq.GetQual(), "rpt_unit_seq")) {
        if (x_IsBaseRange(gbq.GetVal())) {
            gbq.SetQual("rpt_unit_range");
            CGb_qual::CleanupRptUnitRange(gbq.SetVal());
            ChangeMade(CCleanupChange::eChangeQualifiers);
        } else if (CGb_qual::CleanupRptUnitSeq(gbq.SetVal())) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
        x_CleanupRptUnit(gbq);
    } else if (NStr::EqualNocase(gbq.GetQual(), "rpt_unit_range")) {
        if (! x_IsBaseRange(gbq.GetVal())) {
            gbq.SetQual("rpt_unit_seq");
            CGb_qual::CleanupRptUnitSeq(gbq.SetVal());
            ChangeMade(CCleanupChange::eChangeQualifiers);
        } else if (CGb_qual::CleanupRptUnitRange(gbq.SetVal())) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    } else if (NStr::EqualNocase(gbq.GetQual(), "rpt_unit")) {
        if (x_CleanupRptUnit(gbq)) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    } else if (NStr::EqualNocase(gbq.GetQual(), "replace")) {
        if (CGb_qual::CleanupReplace(gbq.SetVal())) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    } else if (NStr::EqualNocase(gbq.GetQual(), "repeat_type")) {
        if (CGb_qual::FixRptTypeValue(gbq.SetVal())) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    } else if (NStr::EqualNocase(gbq.GetQual(), "regulatory_class")) {
        if (CSeqFeatData::FixRegulatoryClassValue(gbq.SetVal())) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    } else if (NStr::EqualNocase(gbq.GetQual(), "pseudogene")) {
        if (CGb_qual::FixPseudogeneValue(gbq.SetVal())) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    }
 
 
    x_ChangeTransposonToMobileElement(gbq);
    x_ChangeInsertionSeqToMobileElement(gbq);
 
    if (NStr::EqualNocase(GET_FIELD(gbq, Qual), "mobile_element")) {
        SET_FIELD( gbq, Qual, "mobile_element_type" );
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
    if (NStr::EqualNocase(gbq.GetQual(), "mobile_element_type") &&
        gbq.IsSetVal() &&
        CGb_qual::FixMobileElementValue(gbq.SetVal())) {
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
}
 
static
const char *s_FindImpFeatType( const CImp_feat &imp )
{
    // keep sorted in ASCII-betical order
    static const char *allowed_types[] = {
        "-10_signal",     "-35_signal",   "3'UTR",          "3'clip",         "5'UTR",
        "5'clip",         "CAAT_signal",  "CDS",            "C_region",       "D-loop",
        "D_segment",      "GC_signal",    "Import",         "J_segment",      "LTR",
        "N_region",       "RBS",          "STS",            "S_region",       "Site-ref",
        "TATA_signal",    "V_region",     "V_segment",      "allele",         "attenuator",
        "centromere",     "conflict",     "enhancer",       "exon",           "gap",
        "iDNA",           "intron",       "mat_peptide",    "misc_RNA",       "misc_binding",
        "misc_difference","misc_feature", "misc_recomb",    "misc_signal",    "misc_structure",
        "mobile_element", "modified_base","mutation",       "old_sequence",   "operon",
        "oriT",           "polyA_signal", "polyA_site",     "precursor_RNA",  "prim_transcript",
        "primer_bind",    "promoter",     "protein_bind",   "regulatory",     "rep_origin",
        "repeat_region",  "repeat_unit",  "satellite",      "sig_peptide",    "source",
        "stem_loop",      "telomere",     "terminator",     "transit_peptide","unsure",
        "variation",      "virion"
    };
    static const int kAllowedTypesNumElems = ( sizeof(allowed_types) / sizeof(allowed_types[0]));
 
    static const char *kFeatBad = "???";
 
    if( ! RAW_FIELD_IS_EMPTY_OR_UNSET(imp, Key) ) {
        // the C logic is more complex than this
        const char *key = GET_FIELD(imp, Key).c_str();
        if( binary_search( allowed_types, allowed_types + kAllowedTypesNumElems,
            key, PCase_CStr() ) )
        {
            return key;
        }
    }
 
    return kFeatBad;
}
 
static
const char *s_FindKeyFromFeatDefType( const CSeq_feat &feat )
{
    static const char *kFeatBad = "???";
    const CSeqFeatData& fdata = feat.GetData();
 
    switch (fdata.Which()) {
        case NCBI_SEQFEAT(Gene):
            return "Gene";
        case NCBI_SEQFEAT(Org):
            return "Org";
        case NCBI_SEQFEAT(Cdregion):
            return "CDS";
        case NCBI_SEQFEAT(Prot):
            if(fdata.GetProt().IsSetProcessed() ) {
                switch( feat.GetData().GetProt().GetProcessed() ) {
                case NCBI_PROTREF(not_set):
                    return "Protein";
                case NCBI_PROTREF(preprotein):
                    return "proprotein";
                case NCBI_PROTREF(mature):
                    return "mat_peptide";
                case NCBI_PROTREF(signal_peptide):
                    return "sig_peptide";
                case NCBI_PROTREF(transit_peptide):
                    return "transit_peptide";
                case NCBI_PROTREF(propeptide):
                    return "propeptide";
                default:
                    return kFeatBad;
                }
            }
            return "Protein";
        case NCBI_SEQFEAT(Rna):
            if(fdata.GetRna().IsSetType() ) {
                const auto& rna = fdata.GetRna();
                switch (rna.GetType() )
                {
                case NCBI_RNAREF(unknown):
                        return "misc_RNA"; // unknownrna mapped to otherrna
                case NCBI_RNAREF(premsg):
                    return "precursor_RNA";
                case NCBI_RNAREF(mRNA):
                    return "mRNA";
                case NCBI_RNAREF(tRNA):
                    return "tRNA";
                case NCBI_RNAREF(rRNA):
                    return "rRNA";
                case NCBI_RNAREF(snRNA):
                    return "snRNA";
                case NCBI_RNAREF(scRNA):
                    return "scRNA";
                case NCBI_RNAREF(snoRNA):
                    return "snoRNA";
                case NCBI_RNAREF(ncRNA):
                    return "ncRNA";
                case NCBI_RNAREF(tmRNA):
                    return "tmRNA";
                case NCBI_RNAREF(miscRNA):
                    return "misc_RNA";
                case NCBI_RNAREF(other):
                    if ( FIELD_IS_SET_AND_IS(rna, Ext, Name) ) {
                        const string &name = rna.GetExt().GetName();
                        if ( NStr::EqualNocase(name, "misc_RNA")) return "misc_RNA";
                        if ( NStr::EqualNocase(name, "ncRNA") ) return "ncRNA";
                        if ( NStr::EqualNocase(name, "tmRNA") ) return "tmRNA";
                    }
                    return "misc_RNA";
                default:
                    return kFeatBad;
                }
            }
            return kFeatBad;
        case NCBI_SEQFEAT(Pub):
            return "Cit";
        case NCBI_SEQFEAT(Seq):
            return "Xref";
        case NCBI_SEQFEAT(Imp):
            return s_FindImpFeatType( fdata.GetImp() );
        case NCBI_SEQFEAT(Region):
            return "Region";
        case NCBI_SEQFEAT(Comment):
            return "Comment";
        case NCBI_SEQFEAT(Bond):
            return "Bond";
        case NCBI_SEQFEAT(Site):
            return "Site";
        case NCBI_SEQFEAT(Rsite):
            return "Rsite";
        case NCBI_SEQFEAT(User):
            return "User";
        case NCBI_SEQFEAT(Txinit):
            return "TxInit";
        case NCBI_SEQFEAT(Num):
            return "Num";
        case NCBI_SEQFEAT(Psec_str):
            return "SecStr";
        case NCBI_SEQFEAT(Non_std_residue):
            return "NonStdRes";
        case NCBI_SEQFEAT(Het):
            return "Het";
        case NCBI_SEQFEAT(Biosrc):
            return "Src";
        case NCBI_SEQFEAT(Clone):
            return "CloneRef";
        case NCBI_SEQFEAT(Variation):
            return "VariationRef";
        default:
            return kFeatBad;
    }
    return kFeatBad;
}
 
 
static bool SetExceptFromGbqual(const CGb_qual& gb_qual, CSeq_feat& feat)
{
    bool rval = false;
    if (!feat.IsSetExcept() || !feat.GetExcept()) {
        feat.SetExcept(true);
        rval = true;
    }
 
    if (!gb_qual.IsSetQual()) {
        return rval;
    }
    if (feat.IsSetExcept_text() && !NStr::IsBlank(feat.GetExcept_text())) {
        return rval;
    }
    // for whatever reason, C Toolkit only sets text if Gbqual was blank
    if (gb_qual.IsSetVal() && !NStr::IsBlank(gb_qual.GetVal())) {
        return rval;
    }
    string exc = gb_qual.GetQual();
    NStr::ReplaceInPlace (exc, "-", " ");
    NStr::ReplaceInPlace (exc, "_", " ");
    feat.SetExcept_text(exc);
    return true;
}
 
 
static bool s_StringsAreEquivalent(const string& str1, const string& str2)
{
    string s1 = NStr::Replace(str1, " ", "_");
    NStr::ReplaceInPlace(s1, "-", "_");
    string s2 = NStr::Replace(str2, " ", "_");
    NStr::ReplaceInPlace(s2, "-", "_");
    return NStr::EqualNocase(s1, s2);
}
 
 
CNewCleanup_imp::EAction CNewCleanup_imp::GBQualSeqFeatBC(CGb_qual& gb_qual, CSeq_feat& feat)
{
    if( ! FIELD_IS_SET(feat, Data) ) {
        return eAction_Nothing;
    }
    CSeqFeatData &data = GET_MUTABLE(feat, Data);
 
    string& qual = GET_MUTABLE(gb_qual, Qual);
    string& val  = GET_MUTABLE(gb_qual, Val);
 
    if( FIELD_EQUALS(feat, Pseudo, false) ) {
        RESET_FIELD(feat, Pseudo);
        ChangeMade (CCleanupChange::eChangeQualifiers);
    }
 
    if( FIELD_EQUALS(feat, Partial, false) ) {
        RESET_FIELD(feat, Partial);
        ChangeMade (CCleanupChange::eChangeQualifiers);
    }
 
    if (NStr::EqualNocase(qual, "cons_splice")) {
        return eAction_Erase;
    } else if (s_StringsAreEquivalent(qual, "ribosomal-slippage") ||
               s_StringsAreEquivalent(qual, "trans-splicing") ||
               s_StringsAreEquivalent(qual, "artificial-location")) {
        if (SetExceptFromGbqual(gb_qual, feat)) {
            ChangeMade (CCleanupChange::eChangeException);
        }
        return eAction_Erase;
    } else if (NStr::EqualNocase(qual, "partial")) {
        feat.SetPartial(true);
        ChangeMade(CCleanupChange::eChangeQualifiers);
        return eAction_Erase;  // mark qual for deletion
    } else if (NStr::EqualNocase(qual, "evidence")) {
        return eAction_Erase;  // mark qual for deletion
    } else if (NStr::EqualNocase(qual, "exception")) {
        if( ! FIELD_EQUALS(feat, Except, true ) ) {
            SET_FIELD(feat, Except, true);
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
        if (!NStr::IsBlank(val)  &&  !NStr::EqualNocase(val, "true")) {
            if (!feat.IsSetExcept_text()) {
                feat.SetExcept_text(val);
                ChangeMade(CCleanupChange::eChangeQualifiers);
            }
        }
        return eAction_Erase;  // mark qual for deletion
    } else if (NStr::EqualNocase(qual, "experiment")) {
        if (NStr::EqualNocase(val, "experimental evidence, no additional details recorded")) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
            return eAction_Erase;  // mark qual for deletion
        }
    } else if (NStr::EqualNocase(qual, "inference")) {
        if (NStr::EqualNocase(val, "non-experimental evidence, no additional details recorded")) {
            ChangeMade(CCleanupChange::eChangeQualifiers);
            return eAction_Erase;  // mark qual for deletion
        } else {
            x_CleanupAndRepairInference(val);
        }
    } else if (NStr::EqualNocase(qual, "note")  ||
               NStr::EqualNocase(qual, "notes")  ||
               NStr::EqualNocase(qual, "comment")) {
        if (!feat.IsSetComment()) {
            feat.SetComment(val);
        } else {
            (feat.SetComment() += "; ") += val;
        }
        ChangeMade(CCleanupChange::eChangeComment);
        ChangeMade(CCleanupChange::eChangeQualifiers);
        return eAction_Erase;  // mark qual for deletion
    } else if( NStr::EqualNocase(qual, "label") ) {
        if ( NStr::EqualNocase(val, s_FindKeyFromFeatDefType(feat)) ) {
            // skip label that is simply the feature key
        } else if ( ! FIELD_IS_SET(feat, Comment) || NStr::FindNoCase(GET_FIELD(feat, Comment), "label") == NPOS) {
            // if label is not already in comment, append
            if( GET_STRING_FLD_OR_BLANK(feat, Comment).empty() ) {
                SET_FIELD(feat, Comment, "label: " + val );
            } else {
                GET_MUTABLE(feat, Comment) += "; label: " + val;
            }
            ChangeMade(CCleanupChange::eChangeComment);
        }
        return eAction_Erase;
    } else if (NStr::EqualNocase(qual, "regulatory_class")) {
        string::size_type colon_pos = val.find_first_of(":");
        if (colon_pos != string::npos && ! NStr::StartsWith (val, "other:")) {
            string comment = val.substr( colon_pos + 1 );
            val.resize( colon_pos );
            if( GET_STRING_FLD_OR_BLANK(feat, Comment).empty() ) {
                SET_FIELD(feat, Comment, comment );
            } else {
                GET_MUTABLE(feat, Comment) += "; " + comment;
            }
            ChangeMade(CCleanupChange::eChangeComment);
        }
    } else if (NStr::EqualNocase(qual, "db_xref")) {
        string tag, db;
        if (NStr::SplitInTwo(val, ":", db, tag)) {
            CRef<CDbtag> dbp(new CDbtag);
            dbp->SetDb(db);
            dbp->SetTag().SetStr(tag);
 
            feat.SetDbxref().push_back(dbp);
            ChangeMade(CCleanupChange::eChangeDbxrefs);
            return eAction_Erase;  // mark qual for deletion
        }
    } else if (NStr::EqualNocase(qual, "gdb_xref")) {
        CRef<CDbtag> dbp(new CDbtag);
        dbp->SetDb("GDB");
        dbp->SetTag().SetStr(val);
        feat.SetDbxref().push_back(dbp);
        ChangeMade(CCleanupChange::eChangeDbxrefs);
        return eAction_Erase;  // mark qual for deletion
    } else if ( NStr::EqualNocase(qual, "pseudo") ) {
        feat.SetPseudo(true);
        ChangeMade(CCleanupChange::eChangeQualifiers);
        return eAction_Erase;  // mark qual for deletion
    } else if ( NStr::EqualNocase(qual, "pseudogene") )
    {
        if( ! FIELD_EQUALS(feat, Pseudo, true) ) {
            feat.SetPseudo(true);
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
 
        // lowercase pseudogene qual
        string new_val = val;
        NStr::ToLower(new_val);
        if( new_val != val ) {
            val = new_val;
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    } else if ( FIELD_IS(data, Gene)  &&  x_GeneGBQualBC( GET_MUTABLE(data, Gene), gb_qual) == eAction_Erase) {
        return eAction_Erase;  // mark qual for deletion
    } else if ( FIELD_IS(data, Cdregion)  &&  x_SeqFeatCDSGBQualBC(feat, GET_MUTABLE(data, Cdregion), gb_qual) == eAction_Erase ) {
        return eAction_Erase;  // mark qual for deletion
    } else if (data.IsRna()  &&  x_SeqFeatRnaGBQualBC(feat, data.SetRna(), gb_qual) == eAction_Erase) {
        return eAction_Erase;  // mark qual for deletion
    } else if (data.IsProt()  &&  x_ProtGBQualBC(data.SetProt(), gb_qual, eGBQualOpt_normal) == eAction_Erase) {
        return eAction_Erase;  // mark qual for deletion
    } else if (NStr::EqualNocase(qual, "gene")) {
        if (!NStr::IsBlank(val)) {
            CRef<CSeqFeatXref> xref(new CSeqFeatXref);
            xref->SetData().SetGene().SetLocus(val);
            feat.SetXref().insert(feat.SetXref().begin(), xref);
            ChangeMade(CCleanupChange::eCopyGeneXref);
            return eAction_Erase;  // mark qual for deletion
        }
    } else if (NStr::EqualNocase(qual, "codon_start")) {
        if (!data.IsCdregion()) {
            // not legal on anything but CDS, so remove it
            return eAction_Erase;  // mark qual for deletion
        }
    } else if ( NStr::EqualNocase(qual, "EC_number") ) {
        x_CleanupECNumber(val);
    } else if( qual == "satellite" ) {
        x_MendSatelliteQualifier( val );
    } else if ( NStr::EqualNocase(qual, "replace") && data.GetSubtype() == CSeqFeatData::eSubtype_variation) {
        string orig = val;
        NStr::ToLower(val);
        if (!NStr::Equal(orig, val)) {
            ChangeMade(CCleanupChange::eCleanQualifiers);
        }
    }
    else if (NStr::EqualNocase(qual, "recombination_class")) {
        if (CGb_qual::FixRecombinationClassValue(val)) {
            ChangeMade(CCleanupChange::eCleanQualifiers);
        }
    }
 
 
    if( NStr::EqualNocase( qual, "mobile_element_type" ) ) {
        // trim spaces around first colon but only if there are no colons
        // with spaces before and after
        if( NPOS != NStr::Find(val, " :") || NPOS != NStr::Find(val, ": ") ) {
            if( s_RegexpReplace( val, "[ ]*:[ ]*", ":", 1 ) ) {
                ChangeMade(CCleanupChange::eCleanQualifiers);
            }
        }
 
        if( data.IsImp() && STRING_FIELD_MATCH( data.GetImp(), Key, "repeat_region" ) && ! val.empty() ) {
            qual = "mobile_element_type";
            data.SetImp().SetKey( "mobile_element" );
            ChangeMade(CCleanupChange::eCleanQualifiers);
        }
    }
 
    // estimated_length must be a number or "unknown"
    if( NStr::EqualNocase( qual, "estimated_length" ) ) {
        if( ! s_IsAllDigits(val) && ! NStr::EqualNocase(val, "unknown") ) {
            val = "unknown";
            ChangeMade(CCleanupChange::eCleanQualifiers);
        }
    }
 
    // conflict is obsolete.  Make it misc_difference, but add a note
    // to the feature comment as to what it used to be.
    if( data.IsImp() && STRING_FIELD_MATCH( data.GetImp(), Key, "conflict" ) ) {
        data.SetImp().SetKey( "misc_difference");
        if( feat.IsSetComment() ) {
            GET_MUTABLE(feat, Comment) = "conflict; " + GET_FIELD(feat, Comment);
        } else {
            SET_FIELD(feat, Comment, "conflict");
        }
        ChangeMade(CCleanupChange::eCleanQualifiers);
    }
 
    if( qual.empty() && val.empty() ) {
        return eAction_Erase;
    }
 
    return eAction_Nothing;
}
 
bool CNewCleanup_imp::x_IsDotBaseRange(const string& val)
{
    size_t pos = NStr::Find(val, "..");
    if (string::npos == pos) {
        return false;
    }
    try {
        long start = NStr::StringToLong(val.substr(0, pos));
        long stop = NStr::StringToLong(val.substr(pos + 2));
        if (start < 1 || stop < 1) {
            return false;
        }
    } catch (...) {
        return false;
    }
    return true;
}
 
 
bool CNewCleanup_imp::x_IsHyphenBaseRange(const string& val)
{
    size_t pos = NStr::Find(val, "-");
    if (string::npos == pos) {
        return false;
    }
    try {
        long start = NStr::StringToLong(val.substr(0, pos));
        long stop = NStr::StringToLong(val.substr(pos + 1));
        if (start < 1 || stop < 1) {
            return false;
        }
    } catch (...) {
        return false;
    }
    return true;
}
 
 
bool CNewCleanup_imp::x_IsBaseRange(const string& val)
{
    if (val.length() > 25) {
        return false;
    }
    if (x_IsDotBaseRange(val)) {
        return true;
    } else if (x_IsHyphenBaseRange(val)) {
        return true;
    } else {
        return false;
    }
}
 
 
bool CNewCleanup_imp::x_CleanupRptUnit(CGb_qual& gbq)
{
    CGb_qual::CleanupRptUnitRange(gbq.SetVal());
    if (x_IsBaseRange(gbq.GetVal())) {
        gbq.SetQual("rpt_unit_range");
        if (x_IsHyphenBaseRange(gbq.GetVal())) {
            NStr::ReplaceInPlace(gbq.SetVal(), "-", "..");
        }
    } else {
        gbq.SetQual("rpt_unit_seq");
        CGb_qual::CleanupRptUnitSeq(gbq.SetVal());
    }
    return true;
}
 
void CNewCleanup_imp::x_ChangeTransposonToMobileElement(CGb_qual& gbq)
//
//  As of Dec 2006, "transposon" is no longer legal as a qualifier. The replacement
//  qualifier is "mobile_element". In addition, the value has to be massaged to
//  indicate "integron" or "transposon".
//
{
    static const string integronValues[] = {
        "class I integron",
        "class II integron",
        "class III integron",
        "class 1 integron",
        "class 2 integron",
        "class 3 integron"
    };
    static const string* endIntegronValues
        = integronValues + sizeof(integronValues)/sizeof(*integronValues);
 
    if (NStr::EqualNocase( GET_FIELD(gbq, Qual), "transposon")) {
        SET_FIELD( gbq, Qual, "mobile_element");
 
        // If the value is one of the IntegronValues, change it to "integron: class XXX":
        const string* pValue = std::find(integronValues, endIntegronValues, GET_FIELD(gbq, Val) );
        if ( pValue != endIntegronValues ) {
            string::size_type cutoff = pValue->find( " integron" );
            _ASSERT( cutoff != string::npos ); // typo in IntegronValues?
            SET_FIELD( gbq, Val, string("integron: ") + pValue->substr(0, cutoff) );
        }
        // Otherwise, just prefix it with "transposon: ":
        else {
            SET_FIELD( gbq, Val, string("transposon: ") + GET_FIELD(gbq, Val) );
        }
 
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
}
 
void CNewCleanup_imp::x_ChangeInsertionSeqToMobileElement(CGb_qual& gbq)
//
//  As of Dec 2006, "insertion_seq" is no longer legal as a qualifier. The replacement
//  qualifier is "mobile_element". In addition, the value has to be massaged to
//  reflect the "insertion_seq".
//
{
    if (NStr::EqualNocase( GET_FIELD(gbq, Qual), "insertion_seq")) {
        gbq.SetQual("mobile_element");
        gbq.SetVal( string("insertion sequence:") + GET_FIELD(gbq, Val) );
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
}
 
static bool s_IsCompoundRptTypeValue(
    const string& value )
//
//  Format of compound rpt_type values: (value[,value]*)
//
//  These are internal to sequin and are in theory cleaned up before the material
//  is released. However, some compound values have escaped into the wild and have
//  not been retro-fixed yet (as of 2006-03-17).
//
{
    if (NStr::IsBlank(value) || value.length() < 3 ||
        !NStr::StartsWith(value, "(") || !NStr::EndsWith(value, ")")) {
        return false;
    }
 
    bool last_char_was_close_paren = false;
    string::const_iterator s = value.begin();
    ++s;
    while (s != value.end()) {
        if (*s == '(') {
            return false;
        } else if (last_char_was_close_paren) {
            return false;
        } else if (*s == ')') {
            last_char_was_close_paren = true;
        }
        ++s;
    }
    return true;
}
 
static
void s_ExpandThisQual(
    CSeq_feat::TQual& quals,        // the list of CGb_qual's.
    CSeq_feat::TQual::iterator& it, // points to the one qual we might expand.
    CSeq_feat::TQual& new_quals )    // new quals that will need to be inserted
//
//  Rules for "rpt_type" qualifiers (as of 2006-03-07):
//
//  There can be multiple occurrences of this qualifier, and we need to keep them
//  all.
//  The value of this qualifier can also be a *list of values* which is *not*
//  conforming to the ASN.1 and thus needs to be cleaned up.
//
//  The cleanup entails turning the list of values into multiple occurrences of the
//  given qualifier, each occurrence taking one of the values in the original
//  list.
//
{
    CGb_qual& qual = **it;
    string  qual_type = qual.GetQual();
    string& val = qual.SetVal();
    if (NStr::Equal(val, "()")) {
        val.clear();
        return;
    }
    if ( ! s_IsCompoundRptTypeValue( val ) ) {
        //
        //  nothing to do ...
        //
        return;
    }
 
    //
    //  Generate list of cleaned up values. Fix original qualifier and generate
    //  list of new qualifiers to be added to the original list:
    //
    vector< string > newValues;
    string valueList = val.substr(1, val.length() - 2);
    NStr::Split(valueList, ",", newValues, NStr::fSplit_Tokenize);
 
    qual.SetVal( newValues[0] );
 
    for ( size_t i=1; i < newValues.size(); ++i ) {
        CRef< CGb_qual > newQual( new CGb_qual() );
        newQual->SetQual( qual_type );
        newQual->SetVal( newValues[i] );
        new_quals.push_back( newQual );
    }
}
 
 
void CNewCleanup_imp::x_ExpandCombinedQuals(CSeq_feat::TQual& quals)
{
    CSeq_feat::TQual    new_quals;
    NON_CONST_ITERATE (CSeq_feat::TQual, it, quals) {
        CGb_qual& gb_qual = **it;
 
        string& qual = GET_MUTABLE(gb_qual, Qual);
        string& val  = GET_MUTABLE(gb_qual, Val);
 
        // convert curly braces to parens for some quals
        if( (val.length() > 1) && (val[0] == '{') &&
            (val[val.length()-1] == '}') )
        {
            val[0] = '(';
            val[val.length()-1] = ')';
            ChangeMade(CCleanupChange::eCleanQualifiers);
        }
 
        if (NStr::EqualNocase(qual, "rpt_type")) {
            s_ExpandThisQual( quals, it, new_quals );
        } else if (NStr::EqualNocase(qual, "rpt_unit")) {
            s_ExpandThisQual( quals, it, new_quals );
        } else if (NStr::EqualNocase(qual, "rpt_unit_range")) {
            s_ExpandThisQual( quals, it, new_quals );
        } else if (NStr::EqualNocase(qual, "rpt_unit_seq")) {
            s_ExpandThisQual( quals, it, new_quals );
        } else if (NStr::EqualNocase(qual, "usedin")) {
            s_ExpandThisQual( quals, it, new_quals );
        } else if (NStr::EqualNocase(qual, "old_locus_tag")) {
            s_ExpandThisQual( quals, it, new_quals );
        } else if (NStr::EqualNocase(qual, "compare")) {
            s_ExpandThisQual( quals, it, new_quals );
        } else if (NStr::EqualNocase(qual, "replace")) {
            s_ExpandThisQual( quals, it, new_quals );
        }
    }
 
    if ( ! new_quals.empty() ) {
        quals.insert(quals.end(), new_quals.begin(), new_quals.end());
        ChangeMade(CCleanupChange::eChangeQualifiers);
        NON_CONST_ITERATE (CSeq_feat::TQual, it, quals) {
            GBQualBC(**it);
        }
    }
}
 
CNewCleanup_imp::EAction
CNewCleanup_imp::x_GeneGBQualBC( CGene_ref& gene, const CGb_qual& gb_qual )
{
    const string& qual = GET_FIELD(gb_qual, Qual);
    const string& val  = GET_FIELD(gb_qual, Val);
 
    if( NStr::IsBlank(val) ) {
        return eAction_Nothing;
    }
 
    bool change_made = false;
    if (NStr::EqualNocase(qual, "map")) {
        if (! gene.IsSetMaploc() ) {
            change_made = true;
            gene.SetMaploc(val);
        }
    } else if (NStr::EqualNocase(qual, "allele")) {
        if ( gene.IsSetAllele() ) {
            return ( NStr::EqualNocase(val, gene.GetAllele()) ? eAction_Erase : eAction_Nothing );
        } else {
            change_made = true;
            gene.SetAllele(val);
        }
    } else if (NStr::EqualNocase(qual, "locus_tag")) {
        if ( ! gene.IsSetLocus_tag() ) {
            change_made = true;
            gene.SetLocus_tag(val);
        }
    } else if (NStr::EqualNocase(qual, "gene_synonym")) {
        change_made = true;
        gene.SetSyn().push_back(val);
    }
    if (change_made) {
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
 
    return ( change_made ? eAction_Erase : eAction_Nothing );
}
 
CNewCleanup_imp::EAction
CNewCleanup_imp::x_SeqFeatCDSGBQualBC(CSeq_feat& feat, CCdregion& cds, const CGb_qual& gb_qual)
{
    const string& qual = gb_qual.GetQual();
    const string& val  = gb_qual.GetVal();
 
    // transl_except qual -> Cdregion.code_break
    if (NStr::EqualNocase(qual, "transl_except")) {
        // could not be parsed earlier
        return eAction_Nothing;
    }
 
    // codon_start qual -> Cdregion.frame
    if (NStr::EqualNocase(qual, "codon_start")) {
        CCdregion::TFrame frame = GET_FIELD(cds, Frame);
        CCdregion::TFrame new_frame = CCdregion::TFrame(NStr::StringToNonNegativeInt(val));
        if (new_frame == CCdregion::eFrame_one  ||
            new_frame == CCdregion::eFrame_two  ||
            new_frame == CCdregion::eFrame_three) {
            if (frame == CCdregion::eFrame_not_set  ||
                ( FIELD_EQUALS( feat, Pseudo, true ) && ! FIELD_IS_SET(feat, Product) )) {
                cds.SetFrame(new_frame);
                ChangeMade(CCleanupChange::eChangeQualifiers);
            }
            return eAction_Erase;
        }
    }
 
    // transl_table qual -> Cdregion.code
    if (NStr::EqualNocase(qual, "transl_table")) {
        if ( FIELD_IS_SET(cds, Code) ) {
            const CCdregion::TCode& code = GET_FIELD(cds, Code);
            int transl_table = 1;
            ITERATE (CCdregion::TCode::Tdata, it, code.Get()) {
                if ( FIELD_IS(**it, Id)  &&  GET_FIELD(**it, Id) != 0) {
                    transl_table = GET_FIELD(**it, Id);
                    break;
                }
            }
 
            if (NStr::EqualNocase(NStr::UIntToString(transl_table), val)) {
                return eAction_Erase;
            }
        } else {
            int new_val = NStr::StringToNonNegativeInt(val);
            if (new_val > 0) {
                CRef<CGenetic_code::C_E> gc(new CGenetic_code::C_E);
                SET_FIELD(*gc, Id, new_val);
                cds.SetCode().Set().push_back(gc);
 
                // we don't have to check except-text because we're
                // setting an unset genetic_code, not changing an existing one
                // (the except-text would be: "genetic code exception")
                ChangeMade(CCleanupChange::eChangeGeneticCode);
                return eAction_Erase;
            }
        }
    }
 
    // look for qualifiers that should be applied to protein feature
    // note - this should be moved to the "indexed" portion of basic cleanup,
    // because it needs to locate another sequence and feature
    if (NStr::Equal(qual, "product") || NStr::Equal (qual, "function") || NStr::EqualNocase (qual, "EC_number")
        || NStr::Equal (qual, "prot_note"))
    {
        // get protein sequence for product
        CRef<CSeq_feat> prot_feat;
        CRef<CProt_ref> prot_ref;
 
        // try to get existing prot_feat
        CBioseq_Handle prot_handle;
        if ( FIELD_IS_SET(feat, Product) ) {
            const CSeq_id *prod_seq_id = feat.GetProduct().GetId();
            if( prod_seq_id ) {
                prot_handle = m_Scope->GetBioseqHandle(*prod_seq_id);
            }
        }
        if (prot_handle) {
            // find main protein feature
            CConstRef<CBioseq> pseq = prot_handle.GetCompleteBioseq();
            if (pseq && pseq->IsSetAnnot()) {
                for (auto ait : pseq->GetAnnot()) {
                    if (ait->IsFtable()) {
                        for (auto fit : ait->GetData().GetFtable()) {
                            if (fit->IsSetData() && fit->GetData().GetSubtype() == CSeqFeatData::eSubtype_prot) {
                                prot_feat.Reset(const_cast<CSeq_feat*>(fit.GetPointer()));
                                prot_ref.Reset(&(prot_feat->SetData().SetProt()));
                            }
                        }
                    }
                }
            }
        }
 
        bool push_back_xref_on_success = false;
        CRef<CSeqFeatXref> xref;
        if ( ! prot_ref ) {
            // otherwise make cross reference
            prot_ref.Reset( new CProt_ref );
 
            // see if this seq-feat already has a prot xref
            EDIT_EACH_SEQFEATXREF_ON_SEQFEAT( xref_iter, feat ) {
                if( (*xref_iter)->IsSetData() && (*xref_iter)->GetData().IsProt() ) {
                    xref = *xref_iter;
                }
            }
            // seq-feat has no prot xref. We make our own.
            if ( ! xref ) {
                xref.Reset( new CSeqFeatXref );
                xref->SetData().SetProt( *prot_ref );
                // we will push the xref onto the feat if the add was successful
                push_back_xref_on_success = true;
            }
            prot_ref.Reset( &xref->SetData().SetProt() );
        }
 
        // replacement prot feature
        EAction action = eAction_Nothing;
 
        if (NStr::Equal(qual, "prot_note") ) {
            if( prot_feat ) {
                if (!prot_feat->IsSetComment() || NStr::IsBlank (prot_feat->GetComment())) {
                    SET_FIELD( *prot_feat, Comment, val);
                } else {
                    SET_FIELD( *prot_feat, Comment, (prot_feat->GetComment() + "; " + val) );
                }
                ChangeMade (CCleanupChange::eChangeComment);
                action = eAction_Erase;
            }
        } else {
            action = x_ProtGBQualBC( *prot_ref, gb_qual, eGBQualOpt_CDSMode );
        }
 
        if( push_back_xref_on_success ) {
            feat.SetXref().push_back( xref );
            ChangeMade(CCleanupChange::eCleanSeqFeatXrefs);
        }
 
        return action;
    }
 
    if (NStr::EqualNocase(qual, "translation")) {
        return eAction_Erase;
    }
 
    return eAction_Nothing;
}
 
typedef SStaticPair<const char *, int> TTrnaKey;
 
static const TTrnaKey trna_key_to_subtype [] = {
    {  "Ala",            'A'  },
    {  "Alanine",        'A'  },
    {  "Arg",            'R'  },
    {  "Arginine",       'R'  },
    {  "Asn",            'N'  },
    {  "Asp",            'D'  },
    {  "Asp or Asn",     'B'  },
    {  "Asparagine",     'N'  },
    {  "Aspartate",      'D'  },
    { "Aspartic",        'D'  },
    { "Aspartic Acid",   'D'  },
    {  "Asx",            'B'  },
    {  "Cys",            'C'  },
    {  "Cysteine",       'C'  },
    {  "fMet",           'M'  },
    {  "Gln",            'Q'  },
    {  "Glu",            'E'  },
    {  "Glu or Gln",     'Z'  },
    {  "Glutamate",      'E'  },
    {  "Glutamic",       'E'  },
    {  "Glutamic Acid",  'E'  },
    {  "Glutamine",      'Q'  },
    {  "Glx",            'Z'  },
    {  "Gly",            'G'  },
    {  "Glycine",        'G'  },
    {  "His",            'H'  },
    {  "Histidine",      'H'  },
    {  "Ile",            'I'  },
    {  "Ile2",           'I'  },
    {  "iMet",           'M'  },
    {  "Isoleucine",     'I'  },
    {  "Leu",            'L'  },
    {  "Leu or Ile",     'J'  },
    {  "Leucine",        'L'  },
    {  "Lys",            'K'  },
    {  "Lysine",         'K'  },
    {  "Met",            'M'  },
    {  "Methionine",     'M'  },
    {  "OTHER",          'X'  },
    {  "Phe",            'F'  },
    {  "Phenylalanine",  'F'  },
    {  "Pro",            'P'  },
    {  "Proline",        'P'  },
    {  "Pyl",            'O'  },
    {  "Pyrrolysine",    'O'  },
    {  "Sec",            'U'  },
    {  "Selenocysteine", 'U'  },
    {  "Ser",            'S'  },
    {  "Serine",         'S'  },
    {  "Ter",            '*'  },
    {  "TERM",           '*'  },
    {  "Termination",    '*'  },
    {  "Thr",            'T'  },
    {  "Threonine",      'T'  },
    {  "Trp",            'W'  },
    {  "Tryptophan",     'W'  },
    {  "Tyr",            'Y'  },
    {  "Tyrosine",       'Y'  },
    {  "Val",            'V'  },
    {  "Valine",         'V'  },
    {  "Xle",            'J'  },
    {  "Xxx",            'X'  }
};
 
typedef CStaticPairArrayMap <const char*, int, PNocase_CStr> TTrnaMap;
DEFINE_STATIC_ARRAY_MAP(TTrnaMap, sm_TrnaKeys, trna_key_to_subtype);
 
// This maps in the opposite direction of sm_TrnaKeys
class CAminoAcidCharToSymbol : public multimap<char, const char*, PNocase_LessChar>
{
public:
    CAminoAcidCharToSymbol( const TTrnaKey keys[], int num_keys )
    {
        int ii = 0;
        for( ; ii < num_keys; ++ii ) {
            insert(value_type( keys[ii].second, keys[ii].first ));
        }
    }
};
const static CAminoAcidCharToSymbol sm_TrnaInverseKeys
    ( trna_key_to_subtype,
      (sizeof(trna_key_to_subtype) / sizeof(trna_key_to_subtype[0])) );
 
static CRef<CTrna_ext> s_ParseTRnaFromAnticodonString (const string &str, const CSeq_feat& feat, CScope *scope)
{
    CRef<CTrna_ext> trna;
 
    if (NStr::IsBlank (str)) return trna;
 
    if (NStr::StartsWith (str, "(pos:")) {
        // find position of closing paren
        string::size_type pos_end = s_MatchingParenPos( str, 0 );
        if (pos_end != string::npos) {
            trna.Reset( new CTrna_ext );
            string pos_str = str.substr (5, pos_end - 5);
            string::size_type aa_start = NStr::FindNoCase (pos_str, "aa:");
            if (aa_start != string::npos) {
                string abbrev = pos_str.substr (aa_start + 3);
                TTrnaMap::const_iterator t_iter = sm_TrnaKeys.find (abbrev.c_str ());
                if (t_iter == sm_TrnaKeys.end ()) {
                    // unable to parse
                    return trna;
                }
                CRef<CTrna_ext::TAa> aa(new CTrna_ext::TAa);
                aa->SetIupacaa (t_iter->second);
                trna->SetAa(*aa);
                pos_str = pos_str.substr (0, aa_start);
                NStr::TruncateSpacesInPlace (pos_str);
                if (NStr::EndsWith (pos_str, ",")) {
                    pos_str = pos_str.substr (0, pos_str.length() - 1);
                }
            }
            const CSeq_loc& loc = feat.GetLocation();
            CRef<CSeq_loc> anticodon = ReadLocFromText (pos_str, loc.GetId(), scope);
            if( anticodon ) {
                CBioseq_Handle bsh = scope->GetBioseqHandle(*(loc.GetId()));
                if (!bsh) {
                    trna.Reset();
                    return trna;
                }
                if (anticodon->GetStop(eExtreme_Positional) >= bsh.GetInst_Length()) {
                    trna.Reset();
                    return trna;
                }
                if (feat.GetLocation().IsSetStrand()) {
                    anticodon->SetStrand(loc.GetStrand());
                } else {
                    anticodon->SetStrand(eNa_strand_plus); // anticodon is always on plus strand
                }
            }
            if (!anticodon) {
                trna->ResetAa();
            } else {
                trna->SetAnticodon(*anticodon);
            }
        }
    }
    return trna;
}
 
static
char s_FindTrnaAA( const string &str )
{
    if ( str.empty() ) return '\0';
    string tmp = str;
    NStr::TruncateSpacesInPlace(tmp);
 
    if( tmp.length() == 1 ) {
        // if the string is a valid one-letter code, just return that
        const char aminoAcidLetter = toupper(tmp[0]);
        if( sm_TrnaInverseKeys.find(aminoAcidLetter) != sm_TrnaInverseKeys.end() ) {
            return aminoAcidLetter;
        }
    } else {
        // translate 3-letter codes and full-names to one-letter codes
        TTrnaMap::const_iterator trna_iter = sm_TrnaKeys.find (tmp.c_str ());
        if( trna_iter != sm_TrnaKeys.end() ) {
            return trna_iter->second;
        }
    }
 
    return '\0';
}
 
class CCharInSet {
public:
    CCharInSet( const string &list_of_characters ) {
        copy( list_of_characters.begin(), list_of_characters.end(),
            inserter( char_set, char_set.begin() ) );
    }
 
    bool operator()( const char ch ) const {
        return ( char_set.find(ch) != char_set.end() );
    }
 
private:
    set<char> char_set;
};
 
static
void s_TokenizeTRnaString (const string &tRNA_string, list<string> &out_string_list )
{
    out_string_list.clear();
    if ( tRNA_string.empty() ) return;
 
    // SGD Tx(NNN)c or Tx(NNN)c#, where x is the amino acid, c is the chromosome (A-P, Q for mito),
    // and optional # is presumably for individual tRNAs with different anticodons and the same
    // amino acid.
    CCachedRegexp valid_sgd_regex = regexpCache.Get(
        "^[Tt][A-Za-z]\\(...\\)[A-Za-z]\\d?\\d?$");
    if ( valid_sgd_regex->IsMatch(tRNA_string) ) {
        // parse SGD tRNA anticodon
        out_string_list.push_back(kEmptyStr);
        string &new_SGD_tRNA_anticodon = out_string_list.back();
        string raw_codon_part = tRNA_string.substr(3,3);
        NStr::ToUpper( raw_codon_part );
        string reverse_complement;
        CSeqManip::ReverseComplement( raw_codon_part, CSeqUtil::e_Iupacna, 0, 3, reverse_complement );
        new_SGD_tRNA_anticodon = string("(") + reverse_complement + ')';
 
        // parse SGD tRNA amino acid
        out_string_list.push_back(tRNA_string.substr(1,1));
        return;
    }
 
    string tRNA_string_copy = tRNA_string;
    // Note that we do NOT remove "*", since it might be a terminator tRNA symbol
    replace_if( tRNA_string_copy.begin(), tRNA_string_copy.end(),
        CCharInSet("-,;:()=\'_~"), ' ' );
 
    vector<string> tRNA_tokens;
    // " \t\n\v\f\r" are the standard whitespace chars
    // ( source: http://www.cplusplus.com/reference/clibrary/cctype/isspace/ )
    NStr::Split(tRNA_string_copy, " \t\n\v\f\r", tRNA_tokens, NStr::fSplit_MergeDelimiters | NStr::fSplit_Truncate);
 
    EDIT_EACH_STRING_IN_VECTOR( tRNA_token_iter, tRNA_tokens ) {
        string &tRNA_token = *tRNA_token_iter;
        // remove initial "tRNA", if any
        if ( NStr::StartsWith(tRNA_token, "tRNA", NStr::eNocase) ) {
            tRNA_token = tRNA_token.substr(4);
        }
        CCachedRegexp threeLettersPlusDigits = regexpCache.Get(
            "^[A-Za-z][A-Za-z][A-Za-z]\\d*$");
        if (! tRNA_token.empty() ) {
            if ( threeLettersPlusDigits->IsMatch(tRNA_token) ) {
                tRNA_token = tRNA_token.substr(0, 3);
            }
            out_string_list.push_back(tRNA_token);
        }
    }
}
 
 
// based on C's ParseTRnaString
static
char s_ParseSeqFeatTRnaString( const string &comment, bool *out_justTrnaText, string &tRNA_codon, bool noSingleLetter )
{
    if (out_justTrnaText) {
        *out_justTrnaText = false;
    }
    tRNA_codon.clear();
 
    if ( comment.empty() ) return '\0';
 
    CRef<CTrna_ext> tr( new CTrna_ext );
 
    char aa = '\0';
    list<string> head;
    s_TokenizeTRnaString (comment, head);
    bool justt = true;
    list<string>::const_iterator head_iter = head.begin();
    bool is_ambig = false;
    for( ; head_iter != head.end(); ++head_iter ) {
        const string &str = *head_iter;
        if( str.empty() ) continue;
        char curraa = '\0';
        if (noSingleLetter && str.length() == 1) {
            curraa = '\0';
        } else {
            curraa = s_FindTrnaAA (str);
        }
        if(curraa != '\0') {
            if (aa == '\0') {
                aa = curraa;
            } else if( curraa != aa) {
                is_ambig = true;
            }
        } else if ( ! NStr::EqualNocase ("tRNA", str) &&
            ! NStr::EqualNocase ("transfer", str) &&
            ! NStr::EqualNocase ("RNA", str) &&
            ! NStr::EqualNocase ("product", str) )
        {
            justt = false;
        }
    }
    if( is_ambig ) {
        aa = 0;
    }
 
    if (justt) {
        if( comment.find_first_of("0123456789") != string::npos ) {
            justt = false;
        }
    }
    if (out_justTrnaText) {
        *out_justTrnaText = justt;
    }
    return aa;
}
 
 
void CNewCleanup_imp::x_AddToComment(CSeq_feat& feat, const string& comment)
{
    if (!feat.IsSetComment()) {
        feat.SetComment(comment);
    }
    else {
        feat.SetComment() += "; " + comment;
    }
    ChangeMade(CCleanupChange::eChangeComment);
}
 
CNewCleanup_imp::EAction
CNewCleanup_imp::x_HandleTrnaProductGBQual(CSeq_feat& feat, CRNA_ref& rna, const string& product)
{
    CRNA_ref::TType& rna_type = rna.SetType();
 
    if (rna_type != CRNA_ref::eType_tRNA &&
        rna_type != CRNA_ref::eType_other &&
        rna_type != CRNA_ref::eType_unknown) {
        return eAction_Nothing;
    }
 
    if (rna_type == NCBI_RNAREF(tRNA) && rna.IsSetExt() && rna.GetExt().IsName()) {
        string name = rna.GetExt().GetName();
        bool justTrnaText = false;
        string codon;
        char aa = s_ParseSeqFeatTRnaString(name, &justTrnaText, codon, false);
        if (aa != '\0') {
            const bool is_fMet = (NStr::Find(name, "fMet") != NPOS);
            const bool is_iMet = (NStr::Find(name, "iMet") != NPOS);
            const bool is_Ile2 = (NStr::Find(name, "Ile2") != NPOS);
            CRNA_ref_Base::C_Ext::TTRNA &trp = rna.SetExt().SetTRNA();
            trp.SetAa().SetNcbieaa(aa);
            if (aa == 'M') {
                if (is_fMet) {
                    x_AddToComment(feat, "fMet");
                } else if (is_iMet) {
                    x_AddToComment(feat, "iMet");
                }
            } else if (aa == 'I') {
                if (is_Ile2) {
                    x_AddToComment(feat, "Ile2");
                }
            }
            x_SeqFeatTRNABC(feat, trp);
            ChangeMade(CCleanupChange::eChangeRNAref);
        }
    }
    if (rna_type == NCBI_RNAREF(tRNA) && !rna.IsSetExt()) {
        // this part inserted from: AddQualifierToFeature (sfp, "product", gb_qual_val);
        bool justTrnaText = false;
        string codon;
        char aa = s_ParseSeqFeatTRnaString(product, &justTrnaText, codon, false);
        if (aa != '\0') {
 
            CRNA_ref_Base::C_Ext::TTRNA& trna = rna.SetExt().SetTRNA();
            trna.SetAa().SetNcbieaa(aa);
 
            if (!justTrnaText || !NStr::IsBlank(codon)) {
                x_AddToComment(feat, product);
            }
 
            if (aa == 'M') {
                if (NStr::Find(product, "fMet") != NPOS &&
                    (!feat.IsSetComment() || NStr::Find(feat.GetComment(), "fMet") == NPOS)) {
                    // x_AddToComment(feat, "fMet");
                    ChangeMade(CCleanupChange::eChangeRNAref);
                    return eAction_Nothing;
                } else if (NStr::Find(product, "iMet") != NPOS &&
                    (!feat.IsSetComment() || NStr::Find(feat.GetComment(), "iMet") == NPOS)) {
                    // x_AddToComment(feat, "iMet");
                    ChangeMade(CCleanupChange::eChangeRNAref);
                    return eAction_Nothing;
                }
            } else if (aa == 'I') {
                if (NStr::Find(product, "Ile2") != NPOS &&
                    (!feat.IsSetComment() || NStr::Find(feat.GetComment(), "Ile2") == NPOS)) {
                    // x_AddToComment(feat, "Ile2");
                    ChangeMade(CCleanupChange::eChangeRNAref);
                    return eAction_Nothing;
                }
            }
 
            ChangeMade(CCleanupChange::eChangeRNAref);
        }
        else {
            x_AddToComment(feat, product);
        }
        return eAction_Erase;
    }
    if (rna_type == NCBI_RNAREF(tRNA) && rna.IsSetExt() && rna.GetExt().IsTRNA()) {
        CRNA_ref_Base::C_Ext::TTRNA& trp = rna.SetExt().SetTRNA();
        if (trp.IsSetAa() && trp.GetAa().IsNcbieaa()) {
            string ignored;
            if (trp.GetAa().GetNcbieaa() == s_ParseSeqFeatTRnaString(product, nullptr, ignored, false) &&
                NStr::IsBlank(ignored)) {
            } else {
                // don't remove product qual because it conflicts with existing aa value
                return eAction_Nothing;
            }
            if (NStr::CompareNocase (product, "tRNA-fMet") == 0 || NStr::CompareNocase (product, "iRNA-fMet") == 0) {
                return eAction_Nothing;
            }
            if (NStr::CompareNocase (product, "tRNA-iMet") == 0 || NStr::CompareNocase (product, "iRNA-iMet") == 0) {
                return eAction_Nothing;
            }
            if (NStr::CompareNocase (product, "tRNA-Ile2") == 0 || NStr::CompareNocase (product, "iRNA-Ile2") == 0) {
                return eAction_Nothing;
            }
            return eAction_Erase;
        } else if (!trp.IsSetAa()) {
            string ignored;
            bool justTrnaText = false;
            char aa = s_ParseSeqFeatTRnaString(product, &justTrnaText, ignored, false);
            if (aa != '\0') {
                trp.SetAa().SetNcbieaa(aa);
                if (!justTrnaText || !NStr::IsBlank(ignored)) {
                    x_AddToComment(feat, product);
                }
                if (NStr::CompareNocase(product, "tRNA-fMet") == 0 ||
                    NStr::CompareNocase(product, "iRNA-fMet") == 0 ||
                    NStr::CompareNocase(product, "tRNA-iMet") == 0 ||
                    NStr::CompareNocase(product, "iRNA-iMet") == 0 ||
                    NStr::CompareNocase(product, "tRNA-Ile2") == 0 ||
                    NStr::CompareNocase(product, "iRNA-Ile2") == 0) {
                    return eAction_Nothing;
                }
                return eAction_Erase;
            }
        }
    }
 
    if (rna.IsSetExt() && rna.GetExt().IsName() && NStr::Equal(rna.GetExt().GetName(), product)) {
        return eAction_Erase;
    }
 
    return eAction_Nothing;
}
 
 
CNewCleanup_imp::EAction CNewCleanup_imp::x_HandleStandardNameRnaGBQual(CSeq_feat& feat, CRNA_ref& rna, const string& standard_name)
{
    return eAction_Nothing;
}
 
 
// homologous to C's HandledGBQualOnRNA.
// That func was copy-pasted, then translated into C++.
// Later we can go back and actually refactor the code
// to make it more efficient or cleaner.
CNewCleanup_imp::EAction
CNewCleanup_imp::x_SeqFeatRnaGBQualBC(CSeq_feat& feat, CRNA_ref& rna, CGb_qual& gb_qual)
{
    if( ! gb_qual.IsSetVal()) {
        return eAction_Nothing;
    }
    const string &gb_qual_qual = gb_qual.GetQual();
    string &gb_qual_val = gb_qual.SetVal();
    TRNAREF_TYPE& rna_type = rna.SetType();
 
    if (NStr::EqualNocase(gb_qual_qual, "standard_name")) {
        return x_HandleStandardNameRnaGBQual(feat, rna, gb_qual_val);
    }
    if (NStr::IsBlank(gb_qual_val)) {
        return eAction_Nothing;
    }
 
    if (NStr::EqualNocase( gb_qual_qual, "product" ))
    {
        if (rna_type == NCBI_RNAREF(unknown)) {
            rna_type = NCBI_RNAREF(other);
            ChangeMade(CCleanupChange::eChangeRNAref);
        }
        if ( rna.IsSetExt() && rna.GetExt().IsName() ) {
            const string &name = rna.SetExt().SetName();
            if ( name.empty() ) {
                rna.ResetExt();
                ChangeMade(CCleanupChange::eChangeRNAref);
            }
        }
        if (x_HandleTrnaProductGBQual(feat, rna, gb_qual_val) == eAction_Erase) {
            return eAction_Erase;
        }
 
        if (!rna.IsSetExt()) {
            string remainder;
            rna.SetRnaProductName(gb_qual_val, remainder);
            ChangeMade(CCleanupChange::eChangeRNAref);
            if (NStr::IsBlank(remainder)) {
                return eAction_Erase;
            } else {
                gb_qual.SetQual(remainder);
                return eAction_Nothing;
            }
        }
        if( rna.GetExt().IsGen() ) {
            CRNA_gen & rna_gen = rna.SetExt().SetGen();
            if( RAW_FIELD_IS_EMPTY_OR_UNSET(rna_gen, Product) ) {
                rna_gen.SetProduct(gb_qual_val);
                ChangeMade(CCleanupChange::eChangeRNAref);
                return eAction_Erase;
            }
            return eAction_Nothing;
        }
        if (rna.GetExt().IsName() && NStr::Equal(rna.GetExt().GetName(), gb_qual_val)) {
            return eAction_Erase;
        }
        if ( rna.IsSetExt() && ! rna.GetExt().IsName() ) return eAction_Nothing;
        const string &name = ( rna.IsSetExt() ? rna.GetExt().GetName() : kEmptyStr );
        if (! name.empty() ) {
            SIZE_TYPE rDNA_pos = NStr::Find( gb_qual_val, "rDNA");
            if (rDNA_pos != NPOS) {
                gb_qual_val[rDNA_pos+1] = 'R';
                ChangeMade(CCleanupChange::eChangeQualifiers);
            }
            if ( NStr::EqualNocase(name, gb_qual_val) ) {
                return eAction_Erase;
            }
            if (rna_type == NCBI_RNAREF(other) || rna_type == NCBI_RNAREF(ncRNA) ||
                rna_type == NCBI_RNAREF(tmRNA) || rna_type == NCBI_RNAREF(miscRNA) )
            {
                // new convention follows ASN.1 spec comments, allows new RNA types
                return eAction_Nothing;
            }
            // subsequent /product now added to comment
            x_AddToComment(feat, gb_qual_val);
            ChangeMade(CCleanupChange::eChangeComment);
            return eAction_Erase;
        }
        if (rna_type == NCBI_RNAREF(ncRNA) ||
            rna_type == NCBI_RNAREF(tmRNA) || rna_type == NCBI_RNAREF(miscRNA) )
        {
            // new convention follows ASN.1 spec comments, allows new RNA types
            return eAction_Nothing;
        }
        if ( ! FIELD_CHOICE_EQUALS( rna, Ext, Name, gb_qual_val) ) {
            rna.SetExt().SetName( gb_qual_val );
            ChangeMade(CCleanupChange::eChangeRNAref);
            return eAction_Erase;
        }
    } else if (NStr::EqualNocase(gb_qual_qual, "anticodon") ) {
        if (!rna.IsSetType() || rna.GetType() == CRNA_ref::eType_unknown) {
            rna.SetType(CRNA_ref::eType_other);
            ChangeMade(CCleanupChange::eChangeKeywords);
        }
        if (rna.GetType() != CRNA_ref::eType_tRNA) {
            return eAction_Nothing;
        }
 
        CRef<CTrna_ext> trna = s_ParseTRnaFromAnticodonString(gb_qual.GetVal(), feat, m_Scope);
        if (!trna) {
            return eAction_Nothing;
        }
 
        x_SeqFeatTRNABC( feat, *trna );
        if (trna->IsSetAa() || trna->IsSetAnticodon()) {
            // don't apply at all if there are conflicts
            bool apply_aa = false;
            bool apply_anticodon = false;
            bool ok_to_apply = true;
 
            // look for conflict with aa
            if (!rna.IsSetExt() || !rna.GetExt().IsTRNA()) {
                if (trna->IsSetAa()) {
                    apply_aa = true;
                }
                if (trna->IsSetAnticodon()) {
                    apply_anticodon = true;
                }
            }
            else {
                if (trna->IsSetAa()) {
                    if (rna.GetExt().GetTRNA().IsSetAa()) {
                        if (rna.GetExt().GetTRNA().GetAa().IsIupacaa()) {
                            if (trna->GetAa().GetIupacaa() != rna.GetExt().GetTRNA().GetAa().GetIupacaa()) {
                                ok_to_apply = false;
                            }
                        }
                    }
                    else {
                        apply_aa = true;
                    }
                }
                // look for conflict with anticodon
                if (trna->IsSetAnticodon()) {
                    if (rna.GetExt().GetTRNA().IsSetAnticodon()) {
                        if (sequence::Compare(rna.GetExt().GetTRNA().GetAnticodon(),
                            trna->GetAnticodon(), m_Scope, sequence::fCompareOverlapping) != sequence::eSame) {
                            ok_to_apply = false;
                        }
                    } else {
                        apply_anticodon = true;
                    }
                }
            }
            if (ok_to_apply) {
                if (apply_aa) {
                    rna.SetExt().SetTRNA().SetAa().SetIupacaa(trna->GetAa().GetNcbieaa());
                    ChangeMade(CCleanupChange::eChange_tRna);
                }
                if (apply_anticodon) {
                    CRef<CSeq_loc> anticodon(new CSeq_loc());
                    anticodon->Add(trna->GetAnticodon());
                    rna.SetExt().SetTRNA().SetAnticodon(*anticodon);
                    ChangeMade(CCleanupChange::eChangeAnticodon);
                }
                return eAction_Erase;
            }
        }
    }
    return eAction_Nothing;
}
 
 
CNewCleanup_imp::EAction
CNewCleanup_imp::x_ProtGBQualBC(CProt_ref& prot, const CGb_qual& gb_qual, EGBQualOpt opt)
{
    const string& qual = gb_qual.GetQual();
    const string& val = gb_qual.GetVal();
 
    if (NStr::EqualNocase(qual, "product") || NStr::EqualNocase(qual, "standard_name")) {
        if (opt == eGBQualOpt_CDSMode || !prot.IsSetName() || NStr::IsBlank(prot.GetName().front())) {
            CCleanup::SetProteinName(prot, val, false);
            ChangeMade(CCleanupChange::eChangeQualifiers);
        } else {
            return eAction_Nothing;
        }
    } else if (NStr::EqualNocase(qual, "function")) {
        ADD_STRING_TO_LIST(prot.SetActivity(), val);
        ChangeMade(CCleanupChange::eChangeQualifiers);
    } else if (NStr::EqualNocase(qual, "EC_number")) {
        ADD_STRING_TO_LIST(prot.SetEc(), val);
        ChangeMade(CCleanupChange::eChangeQualifiers);
    }
 
    // labels to leave alone
    static const char * const ignored_quals[] =
    { "label", "allele", "experiment", "inference", "UniProtKB_evidence",
    "dbxref", "replace", "rpt_unit_seq", "rpt_unit_range" };
    static set<string, PNocase> ignored_quals_raw;
 
    // the mutex is just there in the unlikely event that two separate
    // threads both try to initialized ignored_quals_raw.  It's NOT
    // needed for reading
    static CMutex ignored_quals_raw_initialization_mutex;
    {
        CMutexGuard guard(ignored_quals_raw_initialization_mutex);
        if (ignored_quals_raw.empty()) {
            copy(ignored_quals, ignored_quals + sizeof(ignored_quals) / sizeof(ignored_quals[0]),
                inserter(ignored_quals_raw, ignored_quals_raw.begin()));
        }
    }
 
    if (ignored_quals_raw.find(qual) != ignored_quals_raw.end()) {
        return eAction_Nothing;
    }
 
    // all other gbquals not appropriate on protein features
    return eAction_Erase;
}
 
 
void CNewCleanup_imp::BioSourceEC(CBioSource& biosrc)
{
    x_AddEnvSamplOrMetagenomic(biosrc);
    if (biosrc.IsSetOrg()) {
        x_CleanupOldName(biosrc.SetOrg());
        x_CleanupOrgModNoteEC(biosrc.SetOrg());
    }
}
 
 
void CNewCleanup_imp::x_AddEnvSamplOrMetagenomic(CBioSource& biosrc)
{
    if (!biosrc.IsSetOrg()) {
        return;
    }
    auto& org = biosrc.SetOrg();
    // add environmental_sample or metagenomic based on lineage or div
 
    if ( org.IsSetOrgname()) {
        const auto& orgname = org.GetOrgname();
        bool needs_env_sample = false;
        bool needs_metagenomic = false;
        if (orgname.IsSetLineage()) {
            string lineage = orgname.GetLineage();
            if (NStr::FindNoCase(lineage, "environmental sample") != string::npos) {
                needs_env_sample = true;
            }
            if (NStr::FindNoCase(lineage, "metagenomes") != string::npos) {
                needs_metagenomic = true;
            }
        }
        if (orgname.IsSetDiv()
            && NStr::Equal(biosrc.GetOrg().GetOrgname().GetDiv(), "ENV")) {
            needs_env_sample = true;
        }
 
        if (needs_env_sample || needs_metagenomic) {
            bool has_env_sample = false;
            bool has_metagenomic = false;
            if ( biosrc.IsSetSubtype()) {
                ITERATE(CBioSource::TSubtype, it, biosrc.GetSubtype()) {
                    if ((*it)->IsSetSubtype()) {
                        if ((*it)->GetSubtype() == CSubSource::eSubtype_environmental_sample) {
                            has_env_sample = true;
                        }
                        if ((*it)->GetSubtype() == CSubSource::eSubtype_metagenomic) {
                            has_metagenomic = true;
                        }
                    }
                }
            }
            if (needs_env_sample && !has_env_sample) {
                CRef<CSubSource> s(new CSubSource(CSubSource::eSubtype_environmental_sample, ""));
                biosrc.SetSubtype().push_back(s);
                ChangeMade(CCleanupChange::eAddSubSource);
            }
            if (needs_metagenomic && !has_metagenomic) {
                CRef<CSubSource> s(new CSubSource(CSubSource::eSubtype_metagenomic, ""));
                biosrc.SetSubtype().push_back(s);
                ChangeMade(CCleanupChange::eAddSubSource);
            }
        }
    }
}
 
 
struct SRemovableOldname
{
    const string& m_Taxname;
    bool operator()(CRef<COrgMod> mod)
    {
        return (mod->IsSetSubtype() &&
            mod->GetSubtype() == COrgMod::eSubtype_old_name &&
            mod->IsSetSubname() &&
            NStr::Equal(mod->GetSubname(), m_Taxname) &&
            (!mod->IsSetAttrib() || NStr::IsBlank(mod->GetAttrib())));
    }
};
 
 
void CNewCleanup_imp::x_CleanupOldName(COrg_ref& org)
{
    if (org.IsSetTaxname() && org.IsSetOrgname() && org.GetOrgname().IsSetMod()) {
        SRemovableOldname matcher{ org.GetTaxname() };
        auto& modset = org.SetOrgname().SetMod();
        size_t before = modset.size();
        modset.erase(std::remove_if(modset.begin(), modset.end(), matcher), modset.end());
        if (before != modset.size()) {
            ChangeMade(CCleanupChange::eRemoveOrgmod);
        }
        if (modset.empty()) {
            org.SetOrgname().ResetMod();
            ChangeMade(CCleanupChange::eRemoveOrgmod);
        }
    }
}
 
 
 
bool s_HasMatchingGBMod(const COrgName& org, const string& val)
{
    if (!org.IsSetMod()) {
        return false;
    }
    ITERATE(COrgName::TMod, it, org.GetMod()) {
        if ((*it)->IsSetSubtype() &&
            ((*it)->GetSubtype() == COrgMod::eSubtype_gb_acronym ||
            (*it)->GetSubtype() == COrgMod::eSubtype_gb_anamorph ||
                (*it)->GetSubtype() == COrgMod::eSubtype_gb_synonym) &&
                (*it)->IsSetSubname() &&
            NStr::Equal((*it)->GetSubname(), val)) {
            return true;
        }
    }
    return false;
}
 
 
struct SRemovableOrgModNote {
    const COrg_ref& org;
    bool operator()(CRef<COrgMod> mod) {
        return (mod->IsSetSubtype() &&
            mod->GetSubtype() == COrgMod::eSubtype_other &&
            mod->IsSetSubname() &&
            (s_HasMatchingGBMod(org.GetOrgname(), mod->GetSubname()) ||
            (org.IsSetTaxname() && NStr::Equal(org.GetTaxname(), mod->GetSubname()))));
 
    }
};
 
void CNewCleanup_imp::x_CleanupOrgModNoteEC(COrg_ref& org)
{
    if (!org.IsSetOrgname() || !org.GetOrgname().IsSetMod()) {
        return;
    }
    auto& modset = org.SetOrgname().SetMod();
    SRemovableOrgModNote matcher{ org };
    size_t before = modset.size();
    modset.erase(std::remove_if(modset.begin(), modset.end(), matcher), modset.end());
    if (before != modset.size()) {
        ChangeMade(CCleanupChange::eRemoveOrgmod);
    }
    if (modset.empty()) {
        org.SetOrgname().ResetMod();
        ChangeMade(CCleanupChange::eRemoveOrgmod);
    }
}
 
 
#if 0
void CNewCleanup_imp::x_FlattenPubEquiv(CPub_equiv& pub_equiv)
{
    CPub_equiv::Tdata& data = pub_equiv.Set();
 
    EDIT_EACH_PUB_ON_PUBEQUIV(pub_iter, pub_equiv ) {
        if( FIELD_IS(**pub_iter, Equiv) ) {
            CPub_equiv& equiv = GET_MUTABLE(**pub_iter, Equiv);
            x_FlattenPubEquiv(equiv);
            copy(equiv.Set().begin(), equiv.Set().end(), back_inserter(data));
            ERASE_PUB_ON_PUBEQUIV( pub_iter, pub_equiv );
            ChangeMade(CCleanupChange::eChangePublication);
        }
    }
}
#endif
 
 
void CNewCleanup_imp::x_DateStdBC( CDate_std& date )
{
    if ( FIELD_OUT_OF_RANGE(date, Month, 1, 12) ) {
        RESET_FIELD(date, Month);
        ChangeMade(CCleanupChange::eCleanupDate);
    }
 
    // Maybe we should have the max range set on a per-month basis? (e.g. 30 days for April).
    // ( This could get complex with leap years and such. )
    if ( FIELD_OUT_OF_RANGE(date, Day, 1, 31) ) {
        RESET_FIELD(date, Day);
        ChangeMade(CCleanupChange::eCleanupDate);
    }
 
    if ( FIELD_OUT_OF_RANGE(date, Second, 0, 59) ) {
        RESET_FIELD(date, Second);
        ChangeMade(CCleanupChange::eCleanupDate);
    }
 
    if (date.IsSetMinute()) {
        if (date.GetMinute() < 0 || date.GetMinute() > 59) {
            date.ResetMinute();
            date.ResetSecond();
            ChangeMade(CCleanupChange::eCleanupDate);
        }
    } else if (date.IsSetSecond()) {
        date.ResetSecond();
        ChangeMade(CCleanupChange::eCleanupDate);
    }
 
    if (date.IsSetHour()) {
        if (date.GetHour() < 0 || date.GetHour() > 23) {
            date.ResetHour();
            date.ResetMinute();
            date.ResetSecond();
            ChangeMade(CCleanupChange::eCleanupDate);
        }
    } else if (date.IsSetMinute() || date.IsSetSecond()) {
        date.ResetMinute();
        date.ResetSecond();
        ChangeMade(CCleanupChange::eCleanupDate);
    }
 
}
 
 
void CNewCleanup_imp::x_AddReplaceQual(CSeq_feat& feat, const string& str)
{
    if (!NStr::EndsWith(str, ')')) {
        return;
    }
 
    SIZE_TYPE start = str.find_first_of('\"');
    if (start != NPOS) {
        SIZE_TYPE end = str.find_first_of('\"', start + 1);
        if (end != NPOS) {
            string replace_val = str.substr(start + 1, (end - start) - 1);
            NStr::ToLower(replace_val);
            feat.AddQualifier("replace", replace_val );
            ChangeMade(CCleanupChange::eChangeQualifiers);
        }
    }
}
 
void CNewCleanup_imp::x_SeqIntervalBC( CSeq_interval & seq_interval )
{
    // Fix backwards intervals
    if ( seq_interval.CanGetFrom()  &&  seq_interval.CanGetTo()  &&  seq_interval.GetFrom() > seq_interval.GetTo()) {
        swap(seq_interval.SetFrom(), seq_interval.SetTo());
        ChangeMade(CCleanupChange::eChangeSeqloc);
    }
    // change bad strand values.
    if (m_Scope && seq_interval.IsSetId()) {
        auto seq_type = m_Scope->GetSequenceType(seq_interval.GetId(), CScope::fDoNotRecalculate);
        if (seq_type != CSeq_inst::eMol_not_set) {
            if (CSeq_inst::IsAa(seq_type)) {
                if (seq_interval.IsSetStrand()) {
                    seq_interval.ResetStrand();
                    ChangeMade(CCleanupChange::eChangeStrand);
                }
            } else if (seq_interval.IsSetStrand()) {
                if (seq_interval.GetStrand() == eNa_strand_unknown) {
                    seq_interval.SetStrand(eNa_strand_plus);
                    ChangeMade(CCleanupChange::eChangeStrand);
                }
            } else {
                seq_interval.SetStrand(eNa_strand_plus);
                ChangeMade(CCleanupChange::eChangeStrand);
            }
        }
    }
}
 
void CNewCleanup_imp::x_BothStrandBC( CSeq_loc &loc )
{
    switch (loc.Which()) {
    case CSeq_loc::e_Int :
        x_BothStrandBC( GET_MUTABLE(loc, Int) );
        break;
    case CSeq_loc::e_Packed_int :
        {
            CSeq_loc::TPacked_int::Tdata& ints = loc.SetPacked_int().Set();
            NON_CONST_ITERATE(CSeq_loc::TPacked_int::Tdata, interval_it, ints) {
                x_BothStrandBC(**interval_it);
            }
        }
        break;
    case CSeq_loc::e_Pnt :
        {
            CSeq_loc::TPnt& pnt = loc.SetPnt();
 
            // change both and both-rev to plus and minus, respectively
            if (pnt.CanGetStrand()) {
                ENa_strand strand = pnt.GetStrand();
                if (strand == eNa_strand_both) {
                    pnt.SetStrand(eNa_strand_plus);
                    ChangeMade(CCleanupChange::eChangeStrand);
                } else if (strand == eNa_strand_both_rev) {
                    pnt.SetStrand(eNa_strand_minus);
                    ChangeMade(CCleanupChange::eChangeStrand);
                }
            }
        }
        break;
 
    default:
        break;
    }
}
 
void CNewCleanup_imp::x_BothStrandBC( CSeq_interval & seq_interval )
{
    if (seq_interval.CanGetStrand()) {
        ENa_strand strand = seq_interval.GetStrand();
        if (strand == eNa_strand_both) {
            seq_interval.SetStrand(eNa_strand_plus);
            ChangeMade(CCleanupChange::eChangeStrand);
        } else if (strand == eNa_strand_both_rev) {
            seq_interval.SetStrand(eNa_strand_minus);
            ChangeMade(CCleanupChange::eChangeStrand);
        }
    }
}
 
void CNewCleanup_imp::x_SplitDbtag( CDbtag &dbt, vector< CRef< CDbtag > > & out_new_dbtags )
{
    // check the common case of nothing to split
    if (!dbt.IsSetTag()) {
        return;
    }
    auto& tag = dbt.SetTag();
    if (!tag.IsStr()) {
        return;
    }
    if( tag.GetStr().find(":") == string::npos ) {
        return;
    }
 
    // check if we're trying to split something we shouldn't
    if (dbt.IsSetDb()) {
        string db = dbt.GetDb();
        if (NStr::Equal(db, "MGD") || NStr::Equal(db, "MGI") || NStr::Equal(db, "HGNC") || NStr::Equal(db, "VGNC") || NStr::Equal(db, "AllianceGenome")) {
            return;
        }
    }
 
    if ( m_IsEmblOrDdbj) {
        return;
    }
 
    // split by colon and generate new tags
    vector<string> tags;
    NStr::Split(tag.GetStr(), ":", tags, NStr::fSplit_Tokenize);
    _ASSERT( tags.size() >= 2 );
 
    // treat the CDbtag argument as the first of the new CDbtags
    tag.SetStr( tags.front() );
    vector<string>::const_iterator str_iter = tags.begin() + 1;
    for( ; str_iter != tags.end(); ++str_iter ) {
        CRef<CDbtag> new_tag( new CDbtag );
        new_tag->Assign( dbt );
        new_tag->SetTag().SetStr( *str_iter );
        out_new_dbtags.push_back( new_tag );
    }
 
    ChangeMade(CCleanupChange::eCleanDbtag);
}
 
inline
static
bool s_CodonCompare( const int &codon1, const int &codon2 ) {
    return (codon1 < codon2);
}
 
inline
static
bool s_CodonEqual( int codon1, int codon2 ) {
    return (codon1 == codon2);
}
 
static
char s_ConvertTrnaAaToLetter( const CTrna_ext::C_Aa &trna_aa, CSeqUtil::ECoding coding, char *out_aa_char = nullptr )
{
    char temp_aa = '\0';
 
    size_t num_converted = 0;
    char new_aa = '\0';
    switch( trna_aa.Which() ) {
    case CTrna_ext::C_Aa::e_Iupacaa:
        temp_aa = trna_aa.GetIupacaa();
        num_converted = CSeqConvert::Convert( &temp_aa, CSeqUtil::e_Iupacaa, 0, 1, &new_aa, coding );
        break;
    case CTrna_ext::C_Aa::e_Ncbieaa:
        temp_aa = trna_aa.GetNcbieaa();
        num_converted = CSeqConvert::Convert( &temp_aa, CSeqUtil::e_Ncbieaa, 0, 1, &new_aa, coding );
        break;
    case CTrna_ext::C_Aa::e_Ncbi8aa:
        temp_aa = trna_aa.GetNcbi8aa();
        num_converted = CSeqConvert::Convert( &temp_aa, CSeqUtil::e_Ncbi8aa, 0, 1, &new_aa, coding );
        break;
    case CTrna_ext::C_Aa::e_Ncbistdaa:
        temp_aa = trna_aa.GetNcbistdaa();
        num_converted = CSeqConvert::Convert( &temp_aa, CSeqUtil::e_Ncbistdaa, 0, 1, &new_aa, coding );
        break;
    default:
        break;
    }
    if( out_aa_char ) {
        *out_aa_char = temp_aa;
    }
    if( num_converted > 0 ) {
        return new_aa;
    } else {
        return '\0';
    }
}
 
void CNewCleanup_imp::x_SeqFeatTRNABC( CSeq_feat& feat, CTrna_ext & tRNA )
{
    if( tRNA.IsSetAa() && tRNA.GetAa().IsIupacaa() ) {
        const int old_value = tRNA.GetAa().GetIupacaa();
        tRNA.SetAa().SetNcbieaa( old_value );
        ChangeMade(CCleanupChange::eChange_tRna);
    }
 
    if (! CODON_ON_TRNAEXT_IS_SORTED(tRNA, s_CodonCompare)) {
        SORT_CODON_ON_TRNAEXT(tRNA, s_CodonCompare);
        ChangeMade(CCleanupChange::eChange_tRna);
    }
 
    if( ! CODON_ON_TRNAEXT_IS_UNIQUE(tRNA, s_CodonEqual) ) {
        UNIQUE_CODON_ON_TRNAEXT(tRNA, s_CodonEqual);
        ChangeMade(CCleanupChange::eChange_tRna);
    }
 
    REMOVE_IF_EMPTY_CODON_ON_TRNAEXT(tRNA);
}
 
static
void s_ParsePCRComponent(vector<string> &out_list, const string *component)
{
    out_list.clear();
 
    if( !component ) return;
    if ( component->empty() ) return;
 
    string component_copy = *component; //copy so we can modify it
    // Remove enclosing parens, if any
    const string::size_type len = component_copy.length();
    if ( len > 1 && component_copy[0] == '(' && component_copy[len - 1] == ')' && component_copy.find('(', 1) == string::npos ) {
        component_copy = component_copy.substr( 1, component_copy.length() - 2 );
    }
 
    NStr::Split(component_copy, string(","), out_list, NStr::fSplit_Tokenize);
    EDIT_EACH_STRING_IN_VECTOR( str_iter, out_list ) {
        NStr::TruncateSpacesInPlace( *str_iter );
    }
}
 
class CPCRParsedSet {
public:
    CPCRParsedSet(
        const string* fwd_seq,
        const string* rev_seq,
        const string* fwd_name,
        const string* rev_name) :
    m_Fwd_seq(  fwd_seq ? *fwd_seq : kEmptyStr),
    m_Rev_seq(  rev_seq ? *rev_seq : kEmptyStr),
    m_Fwd_name(fwd_name ? *fwd_name : kEmptyStr),
    m_Rev_name(rev_name ? *rev_name : kEmptyStr),
    m_Original_order(ms_Next_original_order.Add(1)) {}
 
    const string &GetFwdSeq() const { return m_Fwd_seq; }
    const string &GetRevSeq() const { return m_Rev_seq; }
    const string &GetFwdName() const { return m_Fwd_name; }
    const string &GetRevName() const { return m_Rev_name; }
 
    bool operator <( const CPCRParsedSet &rhs ) const {
        if ( int diff = NStr::CompareNocase( m_Fwd_seq, rhs.m_Fwd_seq ) )
            return diff < 0;
        if ( int diff = NStr::CompareNocase( m_Rev_seq, rhs.m_Rev_seq ) )
            return diff < 0;
        if ( int diff = NStr::CompareNocase( m_Fwd_name, rhs.m_Fwd_name ) )
            return diff < 0;
        if ( int diff = NStr::CompareNocase( m_Rev_name, rhs.m_Rev_name ) )
            return diff < 0;
        // last resort
        return m_Original_order < rhs.m_Original_order;
    }
 
private:
    string m_Fwd_seq;
    string m_Rev_seq;
    string m_Fwd_name;
    string m_Rev_name;
    CAtomicCounter::TValue m_Original_order;
 
    static CAtomicCounter ms_Next_original_order;
};
 
CAtomicCounter CPCRParsedSet::ms_Next_original_order;
 
static
void s_ParsePCRSet( const CBioSource &biosrc, list<CPCRParsedSet> &out_pcr_set )
{
    out_pcr_set.clear();
 
    const string* fwd_primer_seq = nullptr;
    const string* rev_primer_seq = nullptr;
    const string* fwd_primer_name = nullptr;
    const string* rev_primer_name = nullptr;
 
// convenience macro
#define PARSEPCRSET_CASE(Subtype) \
            case NCBI_SUBSOURCE(Subtype): \
            if( (*subsrc_iter)->IsSetName() ) { \
                Subtype = &((*subsrc_iter)->GetName()); \
            } \
            break;
 
 
    FOR_EACH_SUBSOURCE_ON_BIOSOURCE( subsrc_iter, biosrc ) {
        SWITCH_ON_SUBSOURCE_CHOICE( **subsrc_iter ) {
        PARSEPCRSET_CASE(fwd_primer_seq)
        PARSEPCRSET_CASE(rev_primer_seq)
        PARSEPCRSET_CASE(fwd_primer_name)
        PARSEPCRSET_CASE(rev_primer_name)
        default:
            // ignore
            break;
        }
    }
#undef PARSEPCRSET_CASE
 
    // ParsePCRStrings
    vector<string> fwd_seq_list;
    s_ParsePCRComponent(fwd_seq_list, fwd_primer_seq);
    vector<string> rev_seq_list;
    s_ParsePCRComponent(rev_seq_list, rev_primer_seq);
    vector<string> fwd_name_list;
    s_ParsePCRComponent(fwd_name_list, fwd_primer_name);
    vector<string> rev_name_list;
    s_ParsePCRComponent(rev_name_list, rev_primer_name);
 
    vector<string>::iterator curr_fwd_seq = fwd_seq_list.begin();
    vector<string>::iterator curr_rev_seq = rev_seq_list.begin();
    vector<string>::iterator curr_fwd_name = fwd_name_list.begin();
    vector<string>::iterator curr_rev_name = rev_name_list.begin();
 
    while (curr_fwd_seq != fwd_seq_list.end() ||
        curr_rev_seq != rev_seq_list.end()    ||
        curr_fwd_name != fwd_name_list.end()  ||
        curr_rev_name != rev_name_list.end() )
    {
        const string* fwd_seq = ( curr_fwd_seq != fwd_seq_list.end() ? &*curr_fwd_seq++ : nullptr );
        const string* rev_seq = ( curr_rev_seq != rev_seq_list.end() ? &*curr_rev_seq++ : nullptr );
        const string* fwd_name = ( curr_fwd_name != fwd_name_list.end() ? &*curr_fwd_name++ : nullptr );
        const string* rev_name = ( curr_rev_name != rev_name_list.end() ? &*curr_rev_name++ : nullptr );
 
        out_pcr_set.push_back( CPCRParsedSet(fwd_seq, rev_seq, fwd_name, rev_name) );
    }
}
 
// split by colon and trim spaces off the pieces
static
void s_ParsePCRColonString( vector<string> &out_list, const string &str )
{
    NStr::Split(str, ":", out_list, NStr::fSplit_Tokenize);
    EDIT_EACH_STRING_IN_VECTOR(str_iter, out_list ) {
        NStr::TruncateSpacesInPlace( *str_iter );
        if( str_iter->empty() ) {
            ERASE_STRING_IN_VECTOR(str_iter, out_list);
        }
    }
}
 
static
CRef<CPCRPrimerSet> s_ModernizePCRPrimerHalf (const string &seq, const string &name)
{
    // Construct the value we will return
    // ( and extract its primer set for easy access )
    CRef<CPCRPrimerSet> return_value( new CPCRPrimerSet );
    list< CRef< CPCRPrimer > > &primer_list = return_value->Set();
 
    vector<string> seq_list;
    s_ParsePCRColonString (seq_list, seq);
    vector<string> name_list;
    s_ParsePCRColonString (name_list, name);
 
    vector<string>::const_iterator name_iter = name_list.begin();
 
    CRef<CPCRPrimer> last_primer;
 
    // create a PCRPrimer for each seq (and attach its name, if possible)
    FOR_EACH_STRING_IN_VECTOR( seq_iter, seq_list ) {
 
        const string* curr_name = nullptr;
        if ( name_iter != name_list.end() ) {
            curr_name = &*name_iter;
            ++name_iter;
        }
 
        CRef<CPCRPrimer> curr_primer( new CPCRPrimer );
        curr_primer->SetSeq().Set( *seq_iter );
        if( curr_name ) {
            curr_primer->SetName().Set( *curr_name );
        }
        primer_list.push_back( curr_primer );
        last_primer = curr_primer;
    }
 
    if( last_primer ) {
        // attach any leftover names to the end of the name of the last seq
        for ( ; name_iter != name_list.end() ; ++name_iter ) {
            last_primer->SetName().Set() += ":" + *name_iter;
        }
    } else {
        // This differs from C.  C breaks as soon as it's looked at the
        // first name, but this version will create CPCRPrimer for all names.
        for ( ; name_iter != name_list.end() ; ++name_iter ) {
            CRef<CPCRPrimer> curr_primer( new CPCRPrimer );
            curr_primer->SetName().Set( *name_iter );
            primer_list.push_back( curr_primer );
        }
    }
 
    // If the CPCRPrimerSet contains nothing inside, return a null ref
    if( primer_list.empty() ) {
        return CRef<CPCRPrimerSet>();
    } else {
        return return_value;
    }
}
 
cla