#include #include #include "../shared/SimpleLogFile.h" #include "../ax_alert_server/AlertConfig.h" #include "../shared/DatabaseForThread.h" #include "Semantics.h" namespace Parse { ///////////////////////////////////////////////////////////////////// // AppropriatePrice::Rule ///////////////////////////////////////////////////////////////////// void AppropriatePrice::Rule::replaceImpl(Tree &tree, Status &status) const { if (tree->as< AppropriatePrice >()) { tree = new PrimaryField(Token(_fieldName), _fieldId); status = sStop; } } ///////////////////////////////////////////////////////////////////// // AppropriatePrice ///////////////////////////////////////////////////////////////////// Tree AppropriatePrice::useCurrentPrice(Tree orig) { AppropriatePrice::Rule rule("price", MainFields::price); return replace(orig, rule); } Tree AppropriatePrice::useLastPrice(Tree orig) { AppropriatePrice::Rule rule("last", MainFields::last); return replace(orig, rule); } ///////////////////////////////////////////////////////////////////// // Math2Helper ///////////////////////////////////////////////////////////////////// class Math2Helper { public: enum Consensus { cOneIsNull, cOneIsNotConst, cDouble, cInteger }; private: Consensus _consensus; int64_t _leftAsInt, _rightAsInt; double _leftAsDouble, _rightAsDouble; public: Math2Helper(Tree left, Tree right); Consensus getConcensus() const { return _consensus; } int64_t getLeftAsInt() const; int64_t getRightAsInt() const; double getLeftAsDouble() const; double getRightAsDouble() const; }; Math2Helper::Math2Helper(Tree left, Tree right) { _consensus = cOneIsNotConst; if (left->as< NullValue >() || right->as< NullValue >()) _consensus = cOneIsNull; else if (IntTreeNode const *leftP = left->as< IntTreeNode >()) { if (IntTreeNode const *rightP = right->as< IntTreeNode >()) { _consensus = cInteger; _leftAsInt = leftP->getValue(); _rightAsInt = rightP->getValue(); } else if (DoubleTreeNode const *rightP = right->as< DoubleTreeNode >()) { _consensus = cDouble; _leftAsDouble = leftP->getValue(); _rightAsDouble = rightP->getValue(); } } else if (DoubleTreeNode const *leftP = left->as< DoubleTreeNode >()) { if (IntTreeNode const *rightP = right->as< IntTreeNode >()) { _consensus = cInteger; _leftAsDouble = leftP->getValue(); _rightAsDouble = rightP->getValue(); } else if (DoubleTreeNode const *rightP = right->as< DoubleTreeNode >()) { _consensus = cDouble; _leftAsDouble = leftP->getValue(); _rightAsDouble = rightP->getValue(); } } } int64_t Math2Helper::getLeftAsInt() const { assert(_consensus == cInteger); return _leftAsInt; } int64_t Math2Helper::getRightAsInt() const { assert(_consensus == cInteger); return _rightAsInt; } double Math2Helper::getLeftAsDouble() const { assert(_consensus == cDouble); return _leftAsDouble; } double Math2Helper::getRightAsDouble() const { assert(_consensus == cDouble); return _rightAsDouble; } ///////////////////////////////////////////////////////////////////// // PrimaryField ///////////////////////////////////////////////////////////////////// // Compare two items. Returns 0 if the items are the same, -1 if the // first item is less than the second item, and 1 if the first item // is greater than the second item. template< typename T > int compare3(T a, T b) { if (a < b) return -1; else if (a > b) return 1; else return 0; } int PrimaryField::compareSameType(TreeNode const &other) const { PrimaryField const &o = dynamic_cast< PrimaryField const & >(other); //PrimaryField const &o = *other.as< PrimaryField >(); return compare3(_fieldId, o._fieldId); } std::string PrimaryField::shortDebug() const { return "PrimaryField(" + ntoa(_fieldId) + ", “" + getToken().getValue() + "”)"; } const Tree PrimaryField::SYMBOL = new PrimaryField(Token("symbol"), MainFields::symbol); const Tree PrimaryField::TIMESTAMP = new PrimaryField(Token("timestamp"), MainFields::timestamp); ///////////////////////////////////////////////////////////////////// // AlertsDaily ///////////////////////////////////////////////////////////////////// AlertsDaily::AlertsDaily(Tree symbol, Tree time) : TreeNode(Token::EMPTY, { symbol, time }) { } Tree AlertsDaily::construct(Args const &children) const { if (children.size() != 2) throw getToken().makeException("Function takes exactly two arguments."); return new AlertsDaily(children[0], children[1]); } Tree AlertsDaily::field(Token const &token, FieldId fieldId) { Tree record = new AlertsDaily(PrimaryField::SYMBOL, PrimaryField::TIMESTAMP); return new SecondaryField(token, fieldId, record); } const Tree AlertsDaily::LIST_EXCH = field(Token("list_exch"), DailyFields::list_exch); ///////////////////////////////////////////////////////////////////// // SecondaryField ///////////////////////////////////////////////////////////////////// int SecondaryField::compareSameType(TreeNode const &other) const { SecondaryField const &o = dynamic_cast< SecondaryField const & >(other); int result = compare3(_fieldId, o._fieldId); return result?result:TreeNode::compareSameType(other); } std::string SecondaryField::shortDebug() const { return "SecondaryField(" + ntoa(_fieldId) + ", “" + getToken().getValue() + "”, " + getRecord()->shortDebug() + ")"; } SecondaryField::SecondaryField(Token const &token, FieldId fieldId, Tree record) : TreeNode(token, { record }), _fieldId(fieldId) { } Tree SecondaryField::construct(Args const &children) const { if (children.size() != 1) // That's a terrible message. The tree requires one child. The other // argument is not stored in the tree. throw getToken().makeException("Function takes exactly one argument."); return new SecondaryField(getToken(), getFieldId(), children[0]); } ///////////////////////////////////////////////////////////////////// // AltField ///////////////////////////////////////////////////////////////////// AltField::AltField(Token token, Args const &children) : TreeNode(token, children) { if (children.size() != 2) throw token.makeException("Function takes exactly two arguments."); if (!getChildren()[0]->as< StringTreeNode >()) throw token.makeException("First argument must be a constant string."); } Tree AltField::create(Token token) { static const Tree ALT_DESCRIPTION = new PrimaryField(Token("alt_description"), MainFields::alt_description); return new AltField(token, token.getValue(), ALT_DESCRIPTION); // I'm pulling out the ALT_DESCRIPTION field and saving it in the tree. // This is our usually trick. If we need this field in more than one // place it will be cached. I did this because it was easy and similar to // what we've done before. I can possibly do better. Ideally we'd save // the parsed version of ALT_DESCRIPTION, not the string value. That would // require an additional Value type. } Tree AltField::construct(Args const &children) const { return new AltField(getToken(), children); } std::string AltField::getFieldName() const { return getChildren()[0]->as< StringTreeNode >()->getValue(); } ///////////////////////////////////////////////////////////////////// // RoundFunction ///////////////////////////////////////////////////////////////////// void RoundFunction::verifyChildren() const { if (getChildren().size() != 2) throw getToken().makeException("round() takes exactly two arguments."); if (!getChildren()[1]->as< IntTreeNode >()) throw getToken().makeException("The second argument to round() must be a constant integer."); } Tree RoundFunction::toRound() const { return getChildren()[0]; } int RoundFunction::digits() const { return getChildren()[1]->as< IntTreeNode >()->getValue(); } ///////////////////////////////////////////////////////////////////// // SectorStringFunction ///////////////////////////////////////////////////////////////////// TreeNode::Args SectorStringFunction::addSymbol(Args const &args) const { if (args.size() == 1) return { PrimaryField::SYMBOL, args[0] }; if (args.size() == 2) return args; throw getToken().makeException("sector_string expects one or two arguments"); } void SectorStringFunction::verifyChildren() const { if (getChildren().size() != 2) throw getToken().makeException("Wrong number of arguments in sector_string."); if (!getChildren()[1]->as< IntTreeNode >()) throw getToken().makeException("The level argument in sector_string must be a constant integer."); } int SectorStringFunction::getLevel() const { return getChildren()[1]->getConstantValue().getInt(); } ///////////////////////////////////////////////////////////////////// // MinFunction ///////////////////////////////////////////////////////////////////// void MinFunction::verifyChildren() const { if (getChildren().size() < 2) throw getToken() .makeException("Function requires at least 2 arguments."); } ///////////////////////////////////////////////////////////////////// // MaxFunction ///////////////////////////////////////////////////////////////////// void MaxFunction::verifyChildren() const { if (getChildren().size() < 2) throw getToken() .makeException("Function requires at least 2 arguments."); } ///////////////////////////////////////////////////////////////////// // IfFunction ///////////////////////////////////////////////////////////////////// TreeNode::Args IfFunction::checkArgs(Token const &token, Args args) { if (args.size() == 3) return args; if (args.size() == 2) { // The default is null. We take care of that here. This tree // node will always have 3 arguments, because we fill in a // default if we need to. args.push_back(NullValue::noToken()); return args; } throw token.makeException("Function requires exactly 2 or 3 arguments."); } Tree IfFunction::optimize(OptimizationContext context) const { const Tree condition = TreeNode::optimize(getCondition(), OptimizationContext::FOR_BOOLEAN); const Tree trueBranch = TreeNode::optimize(getTrueBranch(), context); const Tree falseBranch = TreeNode::optimize(getFalseBranch(), context); if (condition->isConstant()) if (condition->getConstantValue().getBoolean()) return trueBranch; else return falseBranch; else return construct({condition, trueBranch, falseBranch}); } ///////////////////////////////////////////////////////////////////// // ValidDouble ///////////////////////////////////////////////////////////////////// Tree ValidDouble::construct(Args const &children) const { if (children.size() != 1) throw getToken().makeException("Function requires exactly 1 argument."); return new ValidDouble(children[0]); } Tree ValidDouble::optimize(OptimizationContext context) const { const Tree input = TreeNode::optimize(getInput(), OptimizationContext::NORMAL); if (input->isConstant()) { const ValueBox value = input->getConstantValue(); bool valid; double unused; value.getDouble(valid, unused); return valid?IntTreeNode::TRUE:IntTreeNode::FALSE; } return construct({input}); } ///////////////////////////////////////////////////////////////////// // PowFunction ///////////////////////////////////////////////////////////////////// void PowFunction::verifyChildren() const { if (getChildren().size() != 2) throw getToken() .makeException("Function requires exactly 2 arguments."); } ///////////////////////////////////////////////////////////////////// // LogFunction ///////////////////////////////////////////////////////////////////// void LogFunction::verifyChildren() const { if (getChildren().size() != 2) throw getToken() .makeException("Function requires exactly 2 arguments."); } ///////////////////////////////////////////////////////////////////// // Function1 ///////////////////////////////////////////////////////////////////// const std::map< std::string, Function1::Which > Function1::_byName = {{"abs", Function1::Abs}, {"sin", Function1::Sin}, {"cos", Function1::Cos}, {"tan", Function1::Tan}, {"asin", Function1::ASin}, {"acos", Function1::ACos}, {"atan", Function1::ATan}, {"ceil", Function1::Ceil}, {"floor", Function1::Floor}, {"exp", Function1::Exp}, {"ln", Function1::Ln}, {"sqrt", Function1::Sqrt}}; void Function1::verifyChildren() const { if (getChildren().size() != 1) throw getToken().makeException("Function requires exactly 1 argument."); } Function1::Which Function1::getEnum(std::string const &name) { return getProperty(_byName, name, NONE); } Tree Function1::tryCreate(Token const &token, Args const &args) { Which which = getEnum(token.getValue()); if (which == NONE) return NULL; return new Function1(token, which, args); } std::string Function1::shortDebug() const { std::string result = "Function1("; result += ntoa(_which); for (Args::const_iterator it = getChildren().begin(); it != getChildren().end(); it++) { result += ", "; result += (*it)->shortDebug(); } result += ')'; return result; } int Function1::compareSameType(TreeNode const &other) const { Function1 const &o = dynamic_cast< Function1 const & >(other); int result = compare3(_which, o._which); return result?result:TreeNode::compareSameType(other); } ///////////////////////////////////////////////////////////////////// // ComparisonFunction ///////////////////////////////////////////////////////////////////// const std::map< std::string, ComparisonFunction::Which > ComparisonFunction::_byName = {{"<", ComparisonFunction::Less}, {">", ComparisonFunction::Greater}, {"==", ComparisonFunction::Equal}, {"<=", ComparisonFunction::LessOrEqual}, {">=", ComparisonFunction::GreaterOrEqual}, {"!=", ComparisonFunction::NotEqual}}; void ComparisonFunction::verifyChildren() const { if (getChildren().size() != 2) throw getToken().makeException("Function requires exactly 2 arguments."); } Tree ComparisonFunction::tryCreate(Token const &token, Args const &args) { Which which = getProperty(_byName, token.getValue(), (Which)-1); if (which == -1) return NULL; return new ComparisonFunction(token, which, args); } std::string ComparisonFunction::getWhichName() const { for (std::map< std::string, Which >::const_iterator it = _byName.begin(); it != _byName.end(); it++) if (it->second == _which) return it->first; return ntoa(_which); } std::string ComparisonFunction::shortDebug() const { std::string result = "ComparisonFunction("; result += getLHS()->shortDebug(); result += ' '; result += getWhichName(); result += ' '; result += getRHS()->shortDebug(); result += ')'; return result; } int ComparisonFunction::compareSameType(TreeNode const &other) const { ComparisonFunction const &o = dynamic_cast< ComparisonFunction const & >(other); int result = compare3(_which, o._which); return result?result:TreeNode::compareSameType(other); } ///////////////////////////////////////////////////////////////////// // CommonFunctionsRule // // These functions are available to internal and external users. // Starting with +, -, *, and /. ///////////////////////////////////////////////////////////////////// class CommonFunctionsRule : public ReplaceRule { protected: virtual void replaceImpl(Tree &tree, Status &status) const { if (SimpleFunctionNode const *node = tree->as< SimpleFunctionNode >()) { Token const &token = node->getToken(); std::string const &functionName = token.getValue(); if (functionName == "+") { tree = new PlusFunction(token, node->getChildren()); status = sStop; } else if (functionName == "-") { TreeNode::Args const &children = node->getChildren(); switch (children.size()) { case 1: // -1. Turn -x into 0-x just so we have fewer classes. tree = new MinusFunction(token, children[0]); break; case 2: // a - b tree = new MinusFunction(token, children[0], children[1]); break; default: // This shouldn't happen. Probably an error in the parser. throw makeException("Invalid form: " + node->shortDebug()); } status = sStop; } else if (functionName == "*") { tree = new TimesFunction(token, node->getChildren()); status = sStop; } else if (functionName == "/") { TreeNode::Args const &children = node->getChildren(); if (children.size() != 2) throw makeException("Invalid form: " + node->shortDebug()); tree = new DivideFunction(token, children[0], children[1]); status = sStop; } else if ((functionName == "min") || (functionName == "LEAST")) { tree = new MinFunction(token, node->getChildren()); status = sStop; } else if ((functionName == "max") || (functionName == "GREATEST")) { tree = new MaxFunction(token, node->getChildren()); status = sStop; } else if (functionName == "if") { tree = new IfFunction(token, node->getChildren()); status = sStop; } else if ((functionName == "??") || (functionName == "COALESCE")) { tree = new NullCoalescingFunction(token, node->getChildren()); status = sStop; } else if (functionName == "pow") { tree = new PowFunction(token, node->getChildren()); status = sStop; } else if (functionName == "log") { tree = new LogFunction(token, node->getChildren()); status = sStop; } else if (functionName == "&&") { tree = new AndFunction(token, node->getChildren()); status = sStop; } else if (functionName == "||") { tree = new OrFunction(token, node->getChildren()); status = sStop; } else if (functionName == "!") { tree = new NotFunction(token, node->getChildren()); status = sStop; } else if (functionName == "company_name") { tree = CompanyName::create(token, node->getChildren()); status = sStop; } else if (Tree f = Function1::tryCreate(token, node->getChildren())) { tree = f; status = sStop; } else if (Tree f = ComparisonFunction::tryCreate(token, node->getChildren())) { tree = f; status = sStop; } } else if (SimpleItemNode const *node = tree->as< SimpleItemNode >()) { Token const &token = node->getToken(); if (token.getType() == Token::ttSymbol) { std::string value = token.getValue(); if (value == "null") { tree = new NullValue(token); status = sStop; } else if (value == "day_of_week") { tree = DayOfWeek::create(token, node->getChildren()); status = sStop; } else if (value == "seconds_after_open") { tree = SecondsAfterOpen::create(token, node->getChildren()); status = sStop; } else if (value == "company_name") { tree = new CompanyName(token); status = sStop; } else if (value == "standard_deviation") { const Tree price = new AppropriatePrice(token); const Tree volatility = AlertsDaily::field(token, DailyFields::bright_volatility); tree = new TimesFunction(token, price, volatility); status = sStop; } } else if (token.getType() == Token::ttAltField) { tree = AltField::create(token); status = sStop; } } } }; ///////////////////////////////////////////////////////////////////// // InternalFunctionsRule // // These are visible for the built in columns and filters, but not for // the custom formula editor. ///////////////////////////////////////////////////////////////////// class InternalFunctionsRule : public ReplaceRule { protected: virtual void replaceImpl(Tree &tree, Status &status) const { if (SimpleFunctionNode const *node = tree->as< SimpleFunctionNode >()) { Token const &token = node->getToken(); std::string const &functionName = token.getValue(); if (functionName == "round") { // The constructor will verify the arguments / children. tree = new RoundFunction(token, node->getChildren()); status = sStop; } else if (functionName == "sector_string") { // The constructor will verify the arguments / children. tree = new SectorStringFunction(token, node->getChildren()); status = sStop; } } else if (SimpleItemNode const *node = tree->as< SimpleItemNode >()) { Token const &token = node->getToken(); if ((token.getType() == Token::ttSymbol) && (token.getValue() == "$$$")) tree = new AppropriatePrice(token); } // TODO company name. } }; ///////////////////////////////////////////////////////////////////// // AndFunction ///////////////////////////////////////////////////////////////////// bool AndFunction::add(TreeSet &accumulator) const { for (Tree const &child : getChildren()) { bool fastExit = add(accumulator, child); if (fastExit) return true; } return false; } bool AndFunction::add(TreeSet &accumulator, Tree const &child) { if (AndFunction const *asAnd = child->as< AndFunction >()) { // Flatten the tree. The children of this child become the children // of the new item we're going to create. Look for these AndFunction // objects before we optimize. Otherwise the optimize step would create // a lot of unnecessary temporary objects and otherwise waste CPU. bool fastExit = asAnd->add(accumulator); if (fastExit) return true; } else { const Tree newTree = TreeNode::optimize(child, OptimizationContext::FOR_BOOLEAN); if (newTree->isConstant()) { const ValueBox value = newTree->getConstantValue(); if (value.getBoolean() == false) return true; // fast exit! No other inputs matter. // If it was a constant true then we just ignore it. } else if (AndFunction const *andChild = (newTree)->as< AndFunction >()) { // Flatten the tree. The childern of this child become the children // of the new item we're going to create. Don't look for true, false, // or even further flattening opportunities. This child has already // been optimized, so that's been taken care of. for (Tree const &grandChild : andChild->getChildren()) accumulator.insert(grandChild); } else accumulator.insert(newTree); } return false; } Tree AndFunction::optimize(OptimizationContext context) const { //typedef std::set< Tree, TreeCompare > RemoveDuplicates; //typedef std::unordered_set< Tree, TreeHashHelper, TreeHashHelper > // RemoveDuplicates; TreeSet removeDuplicates; bool fastExit = add(removeDuplicates); if (fastExit) return IntTreeNode::FALSE; if (removeDuplicates.size() == 0) // If the arguments are all true (including the case where there are no // arguments) return true. return IntTreeNode::TRUE; if ((context == OptimizationContext::FOR_BOOLEAN) && (removeDuplicates.size() == 1)) // This is a little tricky. Normally when we use &&, we expect all of // the inputs to be booleans and the result will be a boolean. But // that's not always the case. if you say "if(a&&1,x,y)" that is the // same as "if(a,x,y)". In that case "a&&1" is the same as "a" But what // if you just want to print the value "a&&1". In this case "a" is // possibly different. "a" gets converted to a 0 or a 1. This // optimization is important because the C++ code creates a lot of && and // || functions, many of which could go away. But we have to keep the // strict defination to match what someone writing a custom formula // would write. The custom formula interpretation was taken from MySQL. return *removeDuplicates.begin(); std::vector< Tree > newArgs; newArgs.reserve(removeDuplicates.size()); for (Tree const &finalTree : removeDuplicates) newArgs.push_back(finalTree); return construct(newArgs); } ///////////////////////////////////////////////////////////////////// // OrFunction ///////////////////////////////////////////////////////////////////// bool OrFunction::add(TreeSet &accumulator) const { for (Tree const &child : getChildren()) { bool fastExit = add(accumulator, child); if (fastExit) return true; } return false; } bool OrFunction::add(TreeSet &accumulator, Tree const &child) { if (OrFunction const *asOr = child->as< OrFunction >()) { // Flatten the tree. The children of this child become the children // of the new item we're going to create. Look for these OrFunction // objects before we optimize. Otherwise the optimize step would create // a lot of unnecessary temporary objects and otherwise waste CPU. bool fastExit = asOr->add(accumulator); if (fastExit) return true; } else { const Tree newTree = TreeNode::optimize(child, OptimizationContext::FOR_BOOLEAN); if (newTree->isConstant()) { const ValueBox value = newTree->getConstantValue(); if (value.getBoolean() == true) return true; // fast exit! No other inputs matter. // If it was a constant false then we just ignore it. } else if (OrFunction const *orChild = (newTree)->as< OrFunction >()) { // Flatten the tree. The childern of this child become the children // of the new item we're going to create. Don't look for true, false, // or even further flattening opportunities. This child has already // been optimized, so that's been taken care of. for (Tree const &grandChild : orChild->getChildren()) accumulator.insert(grandChild); } else accumulator.insert(newTree); } return false; } Tree OrFunction::optimize(OptimizationContext context) const { TreeSet removeDuplicates; bool fastExit = add(removeDuplicates); if (fastExit) return IntTreeNode::TRUE; if (removeDuplicates.size() == 0) // If the arguments are all false (including the case where there are no // arguments) return false. return IntTreeNode::FALSE; if ((context == OptimizationContext::FOR_BOOLEAN) && (removeDuplicates.size() == 1)) // See comments at this point in AndFunction::optimize() return *removeDuplicates.begin(); std::vector< Tree > newArgs; newArgs.reserve(removeDuplicates.size()); for (Tree const &finalTree : removeDuplicates) newArgs.push_back(finalTree); return construct(newArgs); } ///////////////////////////////////////////////////////////////////// // NotFunction ///////////////////////////////////////////////////////////////////// NotFunction::NotFunction(Token const &token, Args const &children) : TreeNode(token, children) { if (getChildren().size() != 1) throw getToken().makeException("Function requires exactly 1 argument."); } ///////////////////////////////////////////////////////////////////// // SecondsAfterOpen ///////////////////////////////////////////////////////////////////// Tree SecondsAfterOpen::create(Token const &token, Args const &children) { switch (children.size()) { case 0: return new SecondsAfterOpen(token); case 1: return new SecondsAfterOpen(token, children[0]); default: throw token.makeException("Function requires exactly 1 argument."); } } ///////////////////////////////////////////////////////////////////// // DayOfWeek ///////////////////////////////////////////////////////////////////// Tree DayOfWeek::create(Token const &token, Args const &children) { switch (children.size()) { case 0: return new DayOfWeek(token); case 1: return new DayOfWeek(token, children[0]); default: throw token.makeException("Function requires exactly 1 argument."); } } ///////////////////////////////////////////////////////////////////// // CompanyName ///////////////////////////////////////////////////////////////////// Tree CompanyName::create(Token const &token, Args const &children) { switch (children.size()) { case 0: return new CompanyName(token); case 1: return new CompanyName(token, children[0]); default: throw token.makeException("Function requires exactly 1 argument."); } } ///////////////////////////////////////////////////////////////////// // CommonRules ///////////////////////////////////////////////////////////////////// pthread_once_t CommonRules::_hasBeenInitialized = PTHREAD_ONCE_INIT; CommonRules *CommonRules::_instance; void CommonRules::initialize() { _instance = new CommonRules(); } // A subset of the database fields are availble for custom user filters. // The standard filters can use any and all database fields. static std::set< std::string > fieldNamesForCustomFilters = { // alerts "quality", "bid", "ask", "price", "last", "most_recent_close", "expected_open", "t_high", "t_low", "v_up_1", "v_up_2", "v_up_5", "v_up_10", "v_up_15", "v_up_30", "sma_2_5", "sma_2_8", "sma_2_20", "sma_2_200", "sma_5_5", "sma_5_8", "sma_5_20", "sma_5_200", "sma_15_5", "sma_15_8", "sma_15_20", "sma_15_130", "sma_15_200", "sma_60_8", "sma_60_20", "sma_60_200", "std_5_20", "std_15_20", "std_60_20", "vwap", "p_up_1", "v_up_1", "high_pre", "low_pre","limit_up","limit_down","sma_2_10", "sma_5_10", "sma_15_10", "sma_60_10", // alerts_daily "sma_200", "sma_50", "sma_20", "sma_8", "high_p", "low_p", "close_p", "open_p", "volume_p", "high_52w", "low_52w", "std_20", "consolidation_high", "consolidation_low", "last_price", "high_life", "low_life", "high_5d", "low_5d", "close_5d", "high_10d", "low_10d", "close_10d", "high_20d", "low_20d", "close_20d", "adx_14d", "pdi_14d", "mdi_14d", "shares_per_print", "short_interest", "trailing_dividend_rate", "forward_dividend_rate", "social_q", "social_m", "social_w", "social_d", "high_104w", "low_104w", "sma_10", "interest_income" }; CommonRules::CommonRules() : _filterRulesPtr(NULL) { SymbolRules *allDbFields = new SymbolRules; std::vector< DatabaseFieldInfo > const &alertFields = DatabaseFieldInfo::getAlertFields(); for (std::vector< DatabaseFieldInfo >::const_iterator it = alertFields.begin(); it != alertFields.end(); it++) allDbFields->rules[it->databaseName()] = new PrimaryFieldRule(it->id()); std::vector< DatabaseFieldInfo > const &dailyFields = DatabaseFieldInfo::getDailyFields(); for (std::vector< DatabaseFieldInfo >::const_iterator it = dailyFields.begin(); it != dailyFields.end(); it++) allDbFields->rules[it->databaseName()] = new DailyFieldRule(it->id()); RuleList *internalResources = new RuleList; internalResources->rules.push_back(allDbFields); internalResources->rules.push_back(new CommonFunctionsRule); internalResources->rules.push_back(new InternalFunctionsRule); _internalResources = internalResources; updateStandardFilters(); RuleList *publicResources = new RuleList; publicResources->rules.push_back(new CommonFunctionsRule); publicResources->rules.push_back(_filterRules); SymbolRules *publicDbFields = new SymbolRules; for (std::map< std::string, ReplaceRule::Ref >::const_iterator it = allDbFields->rules.begin(); it != allDbFields->rules.end(); it++) if (fieldNamesForCustomFilters.count(it->first)) publicDbFields->rules[it->first] = it->second; publicResources->rules.push_back(publicDbFields); _publicResources = publicResources; } // These should probably eventually move to the .json file. These are the // filters where the SQL code was too complicated for us to parse directly. // So we have overrides here. static const std::map< std::string, std::string > filterOverrides = { { "D_Time", "timestamp" }, { "D_Desc", "description" }, { "D_Sector", "sector_string(2)" }, { "D_SubSector", "sector_string(3)" }, { "D_IndGrp", "sector_string(4)" }, { "D_Industry", "sector_string(5)" }, { "D_SubIndustry", "sector_string(6)" }, { "D_Name", "company_name" }, { "D_Symbol", "symbol" }, { "Time", "seconds_after_open/60.0" } }; void CommonRules::updateStandardFilters() { _filterRulesPtr = new FilterRules; PairedFilterList pairedFilterList (PairedFilterList::ALLOW_RESTRICTED, *DatabaseForThread(DatabaseWithRetry::SLAVE), false, false); for (auto it = pairedFilterList.begin(); it != pairedFilterList.end(); it++) { AllConfigInfo::PairedFilterInfo const &info = **it; TclList msg; const std::string sql = getProperty(filterOverrides, info.baseName, info.sql); msg<assertDone(); msg<<"success"<shortDebug(); _filterRulesPtr->filters[info.baseName] = tree; } catch (Exception &ex) { msg<<"FAIL"<shortDebug(); sendToLogFile(msg); } } _filterRules = _filterRulesPtr; } CommonRules &CommonRules::instance() { if (!_instance) { int result = pthread_once(&_hasBeenInitialized, initialize); assert((result == 0) && _instance); } return *_instance; } ///////////////////////////////////////////////////////////////////// // ConstantStringSet ///////////////////////////////////////////////////////////////////// uint64_t ConstantStringSet::getHashContributaion() const { uint64_t result = _matching.size(); for (std::string const &item : _matching) result ^= sHash(item); return result; } int ConstantStringSet::compareSameType(TreeNode const &other) const { int result = TreeNode::compareSameType(other); if (result) return result; ConstantStringSet const &o = dynamic_cast< ConstantStringSet const & >(other); result = compare3(_matching.size(), o._matching.size()); if (result) return result; for (Matching::const_iterator it1 = _matching.begin(), it2 = o._matching.begin(); it1 != _matching.end(); it1++, it2++) { result = it1->compare(*it2); if (result) return result; } return 0; } Tree ConstantStringSet::construct(Args const &children) const { if (children.size() != 1) throw getToken().makeException("Function requires exactly 1 subtree."); return new ConstantStringSet(getToken(), children[0], _matching); } bool ConstantStringSet::isConstant() const { return _matching.empty(); } ValueBox ConstantStringSet::getConstantValue() const { if (_matching.empty()) return false; else return TreeNode::getConstantValue(); } std::string ConstantStringSet::shortDebug() const { std::string result = "ConstantStringSet("; result += getValue()->shortDebug(); result += ", "; if (_matching.empty()) { // The else case would print "()" for an empty set. We used to do that. // it was correct, but slightly confusing. Now we print the empty set // symbol. result += "∅"; } else { result += "("; bool first = true; for (Matching::const_iterator it = _matching.begin(); it != _matching.end(); it++) { if (first) first = false; else result += ", "; result += "“"; result += *it; result += "”"; } result += ")"; } result += ")"; return result; } ConstantStringSet::ConstantStringSet(Token const &token, Tree value, Matching const &matching) : TreeNode(token, {value}), _matching(matching) { assert(getChildren().size() == 1); } ConstantStringSet::ConstantStringSet(Token const &token, Tree value) : TreeNode(token, {value}) {assert(getChildren().size() ==1); } void ConstantStringSet::add(std::string const &match) { _matching.insert(match); } Tree ConstantStringSet::alertType(Token const &token) { ConstantStringSet *result = new ConstantStringSet(token, new PrimaryField(Token::EMPTY, MainFields::alert_type)); std::vector< std::string > symbols = explode(",", token.getValue()); for (std::vector< std::string >::const_iterator it = symbols.begin(); it != symbols.end(); it++) result->add(*it); return result; } ///////////////////////////////////////////////////////////////////// // CachedValue ///////////////////////////////////////////////////////////////////// __thread std::set< int > *CachedValue::_detailsShown = NULL; int CachedValue::compareSameType(TreeNode const &other) const { CachedValue const &o = dynamic_cast< CachedValue const & >(other); int result = compare3(_location, o._location); return result?result:TreeNode::compareSameType(other); } Tree CachedValue::construct(Args const &children) const { if (children.size() != 1) throw getToken().makeException("Function requires exactly 1 arguments."); return new CachedValue(_location, children[0]); } std::string CachedValue::shortDebug() const { std::string result = "CachedValue("; result += ntoa(_location); result += ", "; bool showDetails = true; if (_detailsShown) { if (_detailsShown->count(_location)) showDetails = false; else _detailsShown->insert(_location); } if (showDetails) result += getCode()->shortDebug(); else result += "…"; result += ')'; return result; } void CachedValue::startDebug() { if (_detailsShown) _detailsShown->clear(); else _detailsShown = new std::set< int >; } void CachedValue::endDebug() { delete _detailsShown; _detailsShown = NULL; } ///////////////////////////////////////////////////////////////////// // AlertRules ///////////////////////////////////////////////////////////////////// void AlertRules::replaceImpl(Tree &tree, Status &status) const { // The database code shows 6 fields that are available to the alerts // but not the top list. The assertDone() function lists at least two // things that are specific to alerts. There's probably some overlap // between these. TODO add all of those items to this rule. if (SimpleItemNode const *node = tree->as< SimpleItemNode >()) { Token const &token = node->getToken(); if (token.getType() == Token::ttAlert) { tree = ConstantStringSet::alertType(token); status = sStop; } } } }