StringStreamCrifLoader Class Reference

#include <orea/simm/crifloader.hpp>

Inheritance diagram for StringStreamCrifLoader:

Collaboration diagram for StringStreamCrifLoader:

Public Member Functions
	StringStreamCrifLoader (const QuantLib::ext::shared_ptr< SimmConfiguration > &configuration, const std::vector< std::set< std::string > > &additionalHeaders={}, bool updateMapper=false, bool aggregateTrades=true, char eol='\n', char delim='\t', char quoteChar='\0', char escapeChar='\\', const std::string &nullString="#N/A")
Public Member Functions inherited from CrifLoader
	CrifLoader (const QuantLib::ext::shared_ptr< SimmConfiguration > &configuration, const std::vector< std::set< std::string > > &additionalHeaders={}, bool updateMapper=false, bool aggregateTrades=true)
virtual	~CrifLoader ()
virtual Crif	loadCrif ()
const QuantLib::ext::shared_ptr< SimmConfiguration > &	simmConfiguration ()
	SIMM configuration getter. More...

Protected Member Functions
Crif	loadCrifImpl () override
Crif	loadFromStream (std::stringstream &&stream)
	Core CRIF loader from generic istream. More...
virtual std::stringstream	stream () const =0
void	processHeader (const std::vector< std::string > &headers)
	Process the elements of a header line of a CRIF file. More...
bool	process (const std::vector< std::string > &entries, QuantLib::Size maxIndex, QuantLib::Size currentLine, Crif &result)
Protected Member Functions inherited from CrifLoader
virtual Crif	loadCrifImpl ()=0
void	addRecordToCrif (Crif &crif, CrifRecord &&recordToAdd) const
void	validateSimmRecord (const CrifRecord &cr) const
	Check if the record is a valid Simm Crif Record. More...
void	currencyOverrides (CrifRecord &crifRecord) const
	Override currency codes. More...
void	updateMapping (const CrifRecord &cr) const
	update bucket mappings More...

Protected Attributes
std::map< QuantLib::Size, QuantLib::Size >	columnIndex_
std::map< QuantLib::Size, std::set< std::string > >	additionalHeadersIndexMap_
char	eol_
char	delim_
char	quoteChar_
char	escapeChar_
std::string	nullString_
Protected Attributes inherited from CrifLoader
QuantLib::ext::shared_ptr< SimmConfiguration >	configuration_
	Simm configuration that is used during loading of CRIF records. More...
std::vector< std::set< std::string > >	additionalHeaders_
	Defines accepted column headers, beyond required and optional headers, see crifloader.cpp. More...
bool	updateMapper_
bool	aggregateTrades_

Additional Inherited Members
Static Protected Attributes inherited from CrifLoader
static std::map< QuantLib::Size, std::set< std::string > >	requiredHeaders
	Map giving required CRIF file headers and their allowable alternatives. More...
static std::map< QuantLib::Size, std::set< std::string > >	optionalHeaders
	Map giving optional CRIF file headers and their allowable alternatives. More...

Detailed Description

Definition at line 101 of file crifloader.hpp.

Constructor & Destructor Documentation

StringStreamCrifLoader()

StringStreamCrifLoader	(	const QuantLib::ext::shared_ptr< SimmConfiguration > &	configuration,
		const std::vector< std::set< std::string > > &	additionalHeaders = {},
		bool	updateMapper = false,
		bool	aggregateTrades = true,
		char	eol = '\n',
		char	delim = '\t',
		char	quoteChar = '\0',
		char	escapeChar = '\\',
		const std::string &	nullString = "#N/A"
	)

Definition at line 197 of file crifloader.cpp.

    : CrifLoader(configuration, additionalHeaders, updateMapper, aggregateTrades), eol_(eol), delim_(delim),
    quoteChar_(quoteChar), escapeChar_(escapeChar), nullString_(nullString) {
    
    size_t maxIndexRequired = *boost::max_element(requiredHeaders | boost::adaptors::map_keys);
    size_t maxIndexOptional = *boost::max_element(optionalHeaders | boost::adaptors::map_keys);
    size_t maxIndex = std::max(maxIndexRequired, maxIndexOptional);
 
    size_t i = 1;
    for (const auto& addHeader : additionalHeaders_) {
        additionalHeadersIndexMap_[maxIndex + i] = addHeader;
        i++;
    }
 
}

Member Function Documentation

loadCrifImpl()

Crif loadCrifImpl ( )

overrideprotectedvirtual

Implements CrifLoader.

Definition at line 109 of file crifloader.hpp.

{ return loadFromStream(stream()); }

Here is the call graph for this function:

loadFromStream()

Crif loadFromStream ( std::stringstream &&  stream )

protected

Core CRIF loader from generic istream.

Definition at line 233 of file crifloader.cpp.

                                                                    {
    string line;
    vector<string> entries;
    bool headerProcessed = false;
    Size emptyLines = 0;
    Size validLines = 0;
    Size invalidLines = 0;
    Size maxIndex = 0;
    Size currentLine = 0;
    Crif result;
    while (getline(stream, line, eol_)) {
 
        // Keep track of current line number for messages
        ++currentLine;
 
        // Trim leading and trailing space
        boost::trim(line);
 
        // Skip empty lines
        if (line.empty()) {
            ++emptyLines;
            continue;
        }
 
        // Break line up in to its elements.
        entries = parseListOfValues(line, escapeChar_, delim_, quoteChar_);
 
        if (headerProcessed) {
            // Process a regular line of the CRIF file
            if (process(entries, maxIndex, currentLine, result)) {
                ++validLines;
            } else {
                ++invalidLines;
            }
        } else {
            // Process the header line of the CRIF file
            processHeader(entries);
            headerProcessed = true;
            auto maxPair = max_element(
                columnIndex_.begin(), columnIndex_.end(),
                [](const pair<Size, Size>& p1, const pair<Size, Size>& p2) { return p1.second < p2.second; });
            maxIndex = maxPair->second;
        }
    }
 
    LOG("Out of " << currentLine << " lines, there were " << validLines << " valid lines, " << invalidLines
                  << " invalid lines and " << emptyLines << " empty lines.");
    return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

stream()

virtual std::stringstream stream ( ) const

protectedpure virtual

Implemented in CsvFileCrifLoader, and CsvBufferCrifLoader.

Here is the caller graph for this function:

processHeader()

void processHeader ( const std::vector< std::string > &  headers )

protected

Process the elements of a header line of a CRIF file.

Definition at line 284 of file crifloader.cpp.

                                                                        {
    columnIndex_.clear();
 
    // Get mapping for all required headers in to column index in the file
    string header;
    for (const auto& kv : requiredHeaders) {
        for (Size i = 0; i < headers.size(); ++i) {
            header = boost::to_lower_copy(headers[i]);
            if (kv.second.count(header) > 0) {
                columnIndex_[kv.first] = i;
            }
        }
        // Some headers are allowed to be missing (under certain circumstances)
        // trade_id, portfolioid and productclass arent required for frtb crif
        if (kv.first == 0 || kv.first == 1 || kv.first == 2) {
            // Portfolio ID header is allowed to be missing
            if (columnIndex_.count(kv.first) == 0) {
                WLOG("Did not find a header for portfolioid in the CRIF file so using a default value");
            }
        } else if (kv.first == 10) {
            // Allow either amount_usd missing or amount and amount_currency, but not all three.
            // For SIMM, we ultimately use amount_usd, but if missing, we use amount and amount_currency
            // and let the SimmCalculator handle the conversion to amount_usd
            if (columnIndex_.count(10) == 0)
                QL_REQUIRE(columnIndex_.count(8) > 0 && columnIndex_.count(9) > 0,
                           "Must provide either amount and amount_currency, or amount_usd");
        } else {
            // All other headers should be there
            QL_REQUIRE(columnIndex_.count(kv.first) > 0,
                       "Could not find a header in the CRIF file for " << *kv.second.begin());
        }
    }
 
    for (const auto& kv : optionalHeaders) {
        for (Size i = 0; i < headers.size(); ++i) {
            header = boost::to_lower_copy(headers[i]);
            if (kv.second.count(header) > 0) {
                columnIndex_[kv.first] = i;
            }
        }
    }    
 
    for (const auto& kv: additionalHeadersIndexMap_) {
        for (Size columnPos = 0; columnPos < headers.size(); ++columnPos) {
            header = boost::to_lower_copy(headers[columnPos]);
            if (kv.second.count(header) > 0) {
                columnIndex_[kv.first] = columnPos;
            }
        }
    }
}

Here is the caller graph for this function:

process()

bool process ( const std::vector< std::string > &  entries, QuantLib::Size  maxIndex, QuantLib::Size  currentLine, Crif &  result  )

protected

Process a line of a CRIF file and return true if valid line or false if an invalid line

Definition at line 336 of file crifloader.cpp.

                                                                                                                 {
    CrifRecord cr;
    // Return early if there are not enough entries in the line
    if (entries.size() <= maxIndex) {
        WLOG("Line number: " << currentLine << ". Expected at least " << maxIndex + 1 << " entries but got only "
                             << entries.size());
        return false;
    }
 
    // Try to create and add a CRIF record
    // There could still be issues here so we surround with try..catch to allow processing to continue
    auto loadOptionalString = [&entries, this](int column) {
        return columnIndex_.count(column) == 0 ? "" : entries[columnIndex_[column]];
    };
    auto loadOptionalReal = [&entries, this](int column) -> QuantLib::Real{
        if (columnIndex_.count(column) == 0) {
            return QuantLib::Null<QuantLib::Real>();
        } else{
            const std::string& value = entries[columnIndex_[column]];
 
            return value.empty() || value == nullString_ ? QuantLib::Null<QuantLib::Real>()
                                                         : parseReal(value);
        } 
    };
 
    string tradeId, tradeType, imModel;
    try {
        tradeId = loadOptionalString(0);
        tradeType = loadOptionalString(15);
        imModel = loadOptionalString(16);
 
        cr.tradeId = tradeId;
        cr.tradeType = tradeType;
        cr.imModel = imModel;
        cr.portfolioId = columnIndex_.count(1) == 0 ? "DummyPortfolio" : entries[columnIndex_.at(1)];
        cr.productClass = parseProductClass(loadOptionalString(2));
        cr.riskType = parseRiskType(entries[columnIndex_.at(3)]);
        
        // Qualifier - There are many other possible qualifier values, but we only do case-insensitive checks
        // for those with standardised values, i.e. currencies or ccy pairs
        cr.qualifier = entries[columnIndex_.at(4)];
        if ((cr.riskType == RiskType::IRCurve || cr.riskType == RiskType::IRVol || cr.riskType == RiskType::FX) &&
            cr.qualifier.size() == 3) {
            string ccyUpper = boost::to_upper_copy(cr.qualifier);
 
            // If ccy is already valid, do nothing. Otherwise, replace with all uppercase equivalent.
            // FIXME: Minor currencies will fail to get spotted here, though it is not likely that we will have
            // a qualifier in a minor ccy?
            if (!checkCurrency(cr.qualifier) && checkCurrency(ccyUpper))
                cr.qualifier = ccyUpper;
        } else if (cr.riskType == RiskType::FXVol && (cr.qualifier.size() == 6 || cr.qualifier.size() == 7)) {
 
            // Remove delimiters between the two currencies
            const string ccyPairDelimiters = "/.,-_|;: ";
            auto ccyPair = ore::data::parseCurrencyPair(boost::to_upper_copy(cr.qualifier), ccyPairDelimiters);
 
            // Convert to uppercase
            string ccy1Upper = ccyPair.first.code();
            string ccy2Upper = ccyPair.second.code();
 
            cr.qualifier = ccy1Upper + ccy2Upper;
        }
 
        // Bucket - Hardcoded "Residual" for case-insensitive check since this is currently the only non-numeric value
        cr.bucket = entries[columnIndex_.at(5)];
        if (boost::to_lower_copy(cr.bucket) == "residual")
            cr.bucket = "Residual";
 
        // Label1
        cr.label1 = entries[columnIndex_.at(6)];
        if (configuration_->isValidRiskType(cr.riskType)) {
            for (const string& l : configuration_->labels1(cr.riskType)) {
                if (boost::to_lower_copy(cr.label1) == boost::to_lower_copy(l))
                    cr.label1 = l;
            }
        }
        // Label2
        cr.label2 = entries[columnIndex_.at(7)];
        if (configuration_->isValidRiskType(cr.riskType)) {
            for (const string& l : configuration_->labels2(cr.riskType)) {
                if (boost::to_lower_copy(cr.label2) == boost::to_lower_copy(l))
                    cr.label2 = l;
            }
        }
 
        // We populate these 'required' values using loadOptional*, but they will have been validated already in processHeader,
        // and missing amountUsd (but with valid amount and amountCurrency) values populated later on in the analytics
 
        cr.amountCurrency = loadOptionalString(8);
        string amountCcyUpper = boost::to_upper_copy(cr.amountCurrency);
        if (!amountCcyUpper.empty() && !checkCurrency(cr.amountCurrency) && checkCurrency(amountCcyUpper))
            cr.amountCurrency = amountCcyUpper;
 
        cr.amount = loadOptionalReal(9);
        cr.amountUsd = loadOptionalReal(10);
 
        // Populate netting set details
        cr.agreementType = loadOptionalString(11);
        cr.callType = loadOptionalString(12);
        cr.initialMarginType = loadOptionalString(13);
        cr.legalEntityId = loadOptionalString(14);
        cr.nettingSetDetails = NettingSetDetails(cr.portfolioId, cr. agreementType, cr.callType, cr.initialMarginType,
                                                 cr.legalEntityId);
        cr.postRegulations = loadOptionalString(17);
        cr.collectRegulations = loadOptionalString(18);
        cr.endDate = loadOptionalString(19);
        cr.label3 = loadOptionalString(20);
        cr.creditQuality = loadOptionalString(21);
        cr.longShortInd = loadOptionalString(22);
        cr.coveredBondInd = loadOptionalString(23);
        cr.trancheThickness = loadOptionalString(24);
        cr.bb_rw = loadOptionalString(25);
 
        // Check the IM model
        try {
            cr.imModel = to_string(parseIMModel(cr.imModel));
        } catch (...) {
            // If we cannot convert to a valid im_model, then it was either provided blank
            // or is simply not a valid value
        }
 
        // Store additional data that matches the defined additional headers in the additional fields map
        for (auto& additionalField : additionalHeadersIndexMap_) {
            std::string value = loadOptionalString(additionalField.first);
            if (!value.empty())
                cr.additionalFields[*additionalField.second.begin()] = value;
        }
 
        // Add the CRIF record to the net records
        addRecordToCrif(result, std::move(cr));
    } catch (const exception& e) {
        ore::data::StructuredTradeErrorMessage(tradeId, tradeType, "CRIF loading",
            "Line number: " + to_string(currentLine) +
                ". Error processing CRIF line, so skipping it. Error: " + to_string(e.what()))
            .log();
        return false;
    }
 
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

columnIndex_

std::map<QuantLib::Size, QuantLib::Size> columnIndex_

protected

Internal map from known index of CRIF record member to file column For example, give trade ID an index of 0 and find the column index of trade ID in the CRIF file e.g. n. The map entry would be [0, n]

Definition at line 119 of file crifloader.hpp.

additionalHeadersIndexMap_

std::map<QuantLib::Size, std::set<std::string> > additionalHeadersIndexMap_

protected

Definition at line 122 of file crifloader.hpp.

eol_

char eol_

protected

Definition at line 131 of file crifloader.hpp.

delim_

char delim_

protected

Definition at line 132 of file crifloader.hpp.

quoteChar_

char quoteChar_

protected

Definition at line 133 of file crifloader.hpp.

escapeChar_

char escapeChar_

protected

Definition at line 134 of file crifloader.hpp.

nullString_

std::string nullString_

protected

Definition at line 135 of file crifloader.hpp.