#include <stdexcept>
#include <algorithm>
#include <limits>

#include "../shared/SimpleLogFile.h"
#include "../shared/ThreadMonitor.h"
#include "../shared/MiscSQL.h"
#include "../shared/DatabaseForThread.h"
#include "AutoSymbolList.h"

#include "Strategy.h"

namespace Parse
{

  /////////////////////////////////////////////////////////////////////
  // SymbolListsWhere
  /////////////////////////////////////////////////////////////////////

  static const std::string s_SEARCH_SYMBOL_LIST = "search_symbol_list";

  class SymbolListsWhere : public TreeNode, public Execution::ICompile
  {
  private:
    class Exe : public Execution::Executable1
    {
    private:
      const AutoSymbolList::Ref _list;
    public:
      Exe(Tree const &symbol, AutoSymbolList::Ref const &list) :
        Executable1(create(symbol)), _list(list) { }
      virtual ValueBox execute(Execution::RecordInfo &recordInfo) const
      {
        const ValueBox symbolBox = _arg1->execute(recordInfo);
        bool valid;
        char const *symbol;
        symbolBox.getString(valid, symbol);
        if (!valid)
          return ValueBox();
        return _list->isPresent(symbol);
      }
    };

    const AutoSymbolList::Ref _list;
    const int _listCount;  // Only used by debugDump().

    SymbolListsWhere(AutoSymbolList::Ref const &list,
                     int listCount,
                     Token const &token,
                     Args args) :
      TreeNode(token, args), _list(list), _listCount(listCount)
    {
      if (getChildren().size() != 1)
        throw getToken().makeException("Function takes exactly one arguments");
    }

  protected:
    virtual int compareSameType(TreeNode const &other) const
    {
      SymbolListsWhere const *const o = other.as< SymbolListsWhere >();
      if (_list < o->_list)
        return -1;
      else if (_list > o->_list)
        return 1;
      else
        return compareChildren(getChildren(), o->getChildren());
    }

    virtual uint64_t getHashContributaion() const { return _list.hash_code(); }


  public:
    static Tree create(std::set< SymbolListDesciption > const &lists,
                       UserId userId,
                       Tree const &symbol = PrimaryField::SYMBOL)
    {
      AutoSymbolList::Description description;
      for (SymbolListDesciption const &sld : lists)
      {
        AutoSymbolList::SingleList l;
        l.ownerId = sld.ownerId;
        if (!l.ownerId)
          l.ownerId = userId;
        l.listId = sld.listId;
        description.insert(l);
      }
      return new SymbolListsWhere(AutoSymbolList::find(description),
                                  description.size(),
                                  symbol->getToken(),
                                  { symbol });
    }

    virtual Tree construct(Args const &children) const
    {
      return new SymbolListsWhere(_list, _listCount, getToken(), children);
    }

    virtual std::string shortDebug() const
    {
      std::string result =
        "SymbolListWhere(" + getSymbol()->shortDebug() + ", ";
      if (_listCount == 1)
        result += "1 list";
      else
      {
        result += ntoa(_listCount);
        result += " lists";
      }
      result += ")";
      // TODO it would be nice to get some debug info from the AutoSymbolList
      // object.  In particular it could say how many symbols are currently
      // present.
      return result;
    }

    Tree getSymbol() const { return getChildren()[0]; }

    virtual Execution::Executable *compile() const
    {
      return new Exe(getSymbol(), _list);
    }
  };

  /////////////////////////////////////////////////////////////////////
  // NewConfigBase
  /////////////////////////////////////////////////////////////////////

  class NewConfigBase
  {
  protected:
    typedef std::vector< std::string > StringList;
  private:
    DatabaseWithRetry &_database;
    const UserId _userId;

    StringList _filterNames;

    typedef std::map< std::string, Tree > AllFormulas;
    // Standard formulas (e.g. [Price]) are immediately filled in.  Custom
    // formulas (e.g. [U37]) initially point to null.  This is used as a cache
    // and the value is filled in the first time it's needed.
    mutable AllFormulas _allFormulas;

    void initFilters()
    {
      assert(_filterNames.empty());
      //AllConfigInfo const &allConfigInfo = AllConfigInfo::instance();
      _allFormulas = CommonRules::instance().standardFilters();
      if (_userId)
      { // Note:  This code is obsolete.  See UserFilters.[Ch] for the
        // new way.  The new was is mostly copied from here, but reorganized
        // to be reused.
        const std::string sql =
          "SELECT internal_code FROM user_filters WHERE user_id="
          + ntoa(_userId);
        for (MysqlResultRef result = _database.tryQueryUntilSuccess(sql);
             result->rowIsValid();
             result->nextRow())
          _allFormulas[result->getStringField(0)] = NULL;
      }
      for (auto const &kvp : _allFormulas)
        _filterNames.push_back(kvp.first);
    }

    Tree getSymbolListsWhere() const
    {
      SymbolLists const &info = getSymbolLists();
      switch (info.getSymbolListType())
      {
      case SymbolLists::sltSingle:
        return
          new ComparisonFunction(ComparisonFunction::Equal,
                                 PrimaryField::SYMBOL,
                                 new StringTreeNode(info.getSingleSymbol()));
      case SymbolLists::sltOnly:
        return SymbolListsWhere::create(info.getSymbolLists(), _userId);
      case SymbolLists::sltExclude:
        return new NotFunction(SymbolListsWhere::create(info.getSymbolLists(),
                                                        _userId));
      case SymbolLists::sltAll:
        return IntTreeNode::TRUE;
      }
      assert(false);
      return NULL;
    }

    static Tree exchangeExpression(BitSet const &exchanges)
    {
      AllConfigInfo const &allConfigInfo = AllConfigInfo::instance();
      if (exchanges == allConfigInfo.getRealExchanges())
        return IntTreeNode::TRUE;
      ConstantStringSet *const set =
        new ConstantStringSet(Token::EMPTY, AlertsDaily::LIST_EXCH);
      for (unsigned bit : exchanges)
        if (AllConfigInfo::ExchangeInfo const *exchange =
            allConfigInfo.getExchange(bit))
          set->add(exchange->shortName);
      return set;
    }

    Tree allowedToShowColumn() const
    {
      const int DELAY = 15 * MARKET_HOURS_MINUTE;
      const Tree time =
        new ComparisonFunction(ComparisonFunction::LessOrEqual,
                               PrimaryField::TIMESTAMP,
                               new MinusFunction(Token::EMPTY,
                                                 new NowFunction,
                                                 IntTreeNode::create(DELAY)));
      // There's room for more optimization here.  exchangeExpression() will
      // return true if all exchanges are allowed.  In this case it's possible,
      // but not likely.  Canada and some others are not allowed except
      // possibly for testing.  However, it's quite reasonable to believe
      // that someone is only asking for a subset of the exchanges (e.g.
      // NASDAQ and NYSE) and he has permission to see columns for all of the
      // exchanges that he's selected (i.e. getActiveExchanges() is a subset of
      // getFullExchangeData()).  In fact, I think that's the common case!
      // TODO
      const Tree exchanges =
        exchangeExpression(getColumnListConfig().getFullExchangeData());
      return new OrFunction({time, exchanges});
    }

  protected:
    UserId getUserId() const { return _userId; }

    static Tree minFilter(Tree filter, double min)
    {
      return new ComparisonFunction(ComparisonFunction::GreaterOrEqual,
                                    filter,
                                    DoubleTreeNode::create(min));
    }
    static Tree maxFilter(Tree filter, double max)
    {
      return new ComparisonFunction(ComparisonFunction::LessOrEqual,
                                    filter,
                                    DoubleTreeNode::create(max));
    }
    static Tree bothFilters(Tree filter, double min, double max)
    {
      if (min == max)
      {
        return new ComparisonFunction(ComparisonFunction::Equal,
                                      filter,
                                      DoubleTreeNode::create(min));
      }
      else if (min < max)
      { // Between min and max, inside.
        return new AndFunction({ minFilter(filter, min),
              maxFilter(filter, max) });
      }
      else
      { // outside.
        return new OrFunction({ minFilter(filter, min),
              maxFilter(filter, max) });
      }
    }

    StringList const &getFilterNames() const { return _filterNames; }

    // This will always return a valid tree.
    Tree getFilterValue(std::string const &name) const
    {
      AllFormulas::iterator it = _allFormulas.find(name);
      if (it == _allFormulas.end())
        // Invalid request.  Do not try to compile.  Return a valid tree that
        // always evaluates to null.
        return new NullValue(Token::EMPTY);
      Tree &result = it->second;
      if (!result)
      { // Find the formula and compile it.
        // Note:  This code is obsolete.  See UserFilters.[Ch] for the
        // new way.  The new was is mostly copied from here, but reorganized
        // to be reused.
        const std::string sql =
          "SELECT source,sql_code='NULL' FROM user_filters WHERE user_id="
          + ntoa(_userId)
          + " AND internal_code='"
          + mysqlEscapeString(name)
          + "'";
        MysqlResultRef row = _database.tryQueryUntilSuccess(sql);
        if (!row->rowIsValid())
          result = new NullValue(Token::EMPTY);
        else
          try
          {
            Parser parser(row->getStringField(0));
            result = parser.parse();
            result =
              replace(result, *CommonRules::instance().publicResources());
            result->assertDone();
          }
          catch (Exception const &ex)
          {
            result = new NullValue(Token::EMPTY);
            if (!row->getBooleanField(1))
            { // The PHP/MySQL version was able to parse this, but we could
              // not.
              TclList msg;
              msg<<FLF<<"UNABLE TO PARSE"
                 <<"_userId"<<_userId
                 <<"name"<<name
                 <<"message"<<ex.getMessage()
                 <<"location"<<ex.getLocation();
              sendToLogFile(msg);
            }
          }
      }
      return result;
    }

    NewConfigBase(DatabaseWithRetry &database, UserId userId) :
      _database(database), _userId(userId)
    {
      initFilters();
    }

    // Anything that is specific to the alerts or top list.  The window
    // specific filters are stored in a different way for these two.  Also,
    // only the alert window has alerts.  This does not include the exchanges
    // or the symbol lists, because those are handled elsewhere.  Those are
    // so similar for both types of window, so the conversion from the
    // underlying data to a Tree is handled in this class, rather than each
    // sub-class.
    virtual Tree getWhereOther() const =0;

    virtual BitSet const &getActiveExchanges() const =0;
    virtual SymbolLists const &getSymbolLists() const =0;
    virtual ColumnListConfig const &getColumnListConfig() const =0;
  public:
    Tree getWhere() const
    {
      TreeNode::Args resultSoFar;
      resultSoFar.push_back(getSymbolListsWhere());
      resultSoFar.push_back(getWhereOther());
      resultSoFar.push_back(exchangeExpression(getActiveExchanges()));
      return new AndFunction(resultSoFar);
    }
    virtual Tree getSort() const =0;

    // The base version of getColumns() will return exactly the  columns that
    // the user asked for in the columns tab of the config window.  That may be
    // customized.  If I remember correctly, the alert window always sends the
    // "SYMBOL" but the top list window calls that data the "symbol".
    virtual ColumnsByName getColumns() const
    {
      ColumnsByName result;
      ColumnListConfig const &columnListConfig = getColumnListConfig();
      Tree allowed = allowedToShowColumn();
      for (ColumnListConfig::Mapping const &mapping :
             columnListConfig.getMappings())
      {
        Tree column = getFilterValue(mapping.internalCode);
        if (!mapping.alwaysShow)
          // For this column only show prices if we have delayed data.
          column = new IfFunction(Token::EMPTY, { allowed, column });
        result[mapping.altName] = column;
      }
      return result;
    }
    StrategyTrees getStrategyTrees() const
    {
      return StrategyTrees(getWhere(), getSort(), getColumns());
    }
  };


  /////////////////////////////////////////////////////////////////////
  // NewAlertConfig
  /////////////////////////////////////////////////////////////////////

  class NewAlertConfig : public AlertConfig, public NewConfigBase
  {
  private:
    static Tree getAlertType()
    {
      static const Tree ALERT_TYPE =
        new PrimaryField(Token("alert_type"), MainFields::alert_type);
      return ALERT_TYPE;
    }
    Tree getWhereAlertTypes() const
    {
      AllConfigInfo const &allConfigInfo = AllConfigInfo::instance();
      BitSet const &activeAlerts = getActiveAlerts();
      static const Tree QUALITY =
        new PrimaryField(Token("quality"), MainFields::quality);
      ConstantStringSet *const set =
        new ConstantStringSet(Token::EMPTY, getAlertType());
      TreeNode::Args result;
      for (unsigned bit : activeAlerts)
        if (AllConfigInfo::AlertInfo const *alert =
            allConfigInfo.getAlert(bit))
        {
          if (SimpleExpression const *quality =
              getProperty(getAlertQuality(), bit))
          {
            const Tree a =
              new ComparisonFunction(ComparisonFunction::Equal,
                                     getAlertType(),
                                     new StringTreeNode(alert->shortName));
            const Tree b =
              new ComparisonFunction(ComparisonFunction::GreaterOrEqual,
                                     QUALITY,
                                     DoubleTreeNode::create(quality->getValue()));
            result.push_back(new AndFunction({a, b}));
          }
          else
          {
            set->add(alert->shortName);
          }
        }
      result.push_back(set);
      return new OrFunction(result);
    }
  protected:
    virtual Tree getWhereOther() const
    {
      std::vector< Tree > resultSoFar;
      resultSoFar.push_back(getWhereAlertTypes());
      for (std::string const &name : getFilterNames())
      {
        SimpleExpression const *minValue =
          getProperty(getWindowFilters(), "Min" + name);
        SimpleExpression const *maxValue =
          getProperty(getWindowFilters(), "Max" + name);
        if (minValue)
        {
          if (maxValue)
          {
            resultSoFar.push_back(bothFilters(getFilterValue(name),
                                              minValue->getValue(),
                                              maxValue->getValue()));
          }
          else
          {
            resultSoFar.push_back(minFilter(getFilterValue(name),
                                            minValue->getValue()));
          }
        }
        else
        {
          if (maxValue)
            resultSoFar.push_back(maxFilter(getFilterValue(name),
                                            maxValue->getValue()));
        }
      }
      return new AndFunction(resultSoFar);
    }
    virtual BitSet const &getActiveExchanges() const
    { return AlertConfig::getActiveExchanges(); }
    virtual SymbolLists const &getSymbolLists() const
    { return AlertConfig::getSymbolLists(); }
    virtual ColumnListConfig const &getColumnListConfig() const
    { return AlertConfig::getColumnListConfig(); }
  public:
    NewAlertConfig(DatabaseWithRetry &database, UserId userId) :
      NewConfigBase(database, userId) { }
    virtual Tree getSort() const
    {
      return NullValue::noToken();
    }
    virtual ColumnsByName getColumns() const
    {
      static Tree TIME_FIELD =
        new PrimaryField(Token("time"), MainFields::timestamp);
      static Tree QUALITY_FIELD =
        new PrimaryField(Token("quality"), MainFields::quality);
      static Tree FOUR_DIGITS = 
	Parse::replace(Parse::Parser("price<1.0").parse(),
		       *Parse::CommonRules::instance().publicResources());
      ColumnsByName result = NewConfigBase::getColumns();
      if (result.count("c_D_Desc"))
      { // If the client asks for the description, also send them the
        // ALT_DESCRIPTION field and the quality field.  The former is only
        // available this way.  The user might for the quality field.  In
        // that case nothing changes.  He will only get one copy of that field.
        result["ALT_DESCRIPTION"] =
          new PrimaryField(Token::EMPTY, MainFields::alt_description);
        result["c_D_Quality"] = QUALITY_FIELD;
      }
      // Always include the alert type.  The client expects that.  If the user
      // asked to see this field then it will aready be here.  The following
      // line won't make any difference in that case.
      result["c_D_Type"] = getAlertType();
      // Same for a few more fields.
      result["c_D_Symbol"] = PrimaryField::SYMBOL;
      result["c_D_Exch"] = AlertsDaily::LIST_EXCH;
      result["c_D_Time"] = TIME_FIELD;
      result["four_digits"] = FOUR_DIGITS;
      //TclList msg;
      //msg<<FLF;
      //for (auto it = result.begin(); it != result.end(); it++)
      //msg<<(TclList()<<it->first<<it->second->shortDebug());
      //sendToLogFile(msg);
      return result;
    }
  };


  /////////////////////////////////////////////////////////////////////
  // NewTopListConfig
  /////////////////////////////////////////////////////////////////////

  class NewTopListConfig : public TopListConfig, public NewConfigBase,
                           public ITopListConfigFromCollaborate
  {
  private:
    Tree getSortField() const { return getFilterValue(getSortBy().field()); }

    // If this is true, automatically add some extra columns, like symbol
    // and listed exchange.  If this is false, only add what was explicitly
    // requested.
    bool _addMagicColumns;

  protected:
    virtual Tree getWhereOther() const
    {
      TreeNode::Args resultSoFar;
      if (!hasSingleSymbol())
        // Traditionally if you sort by a field that automatically adds a
        // where condition.  That field must be valid.  That makes some sense
        // in general.  That's especially important in the C++ code, because if
        // you try to sort by NaN you might get a core dump!  The problem is
        // that sometimes you want one specific stock.  For the most part you
        // don't want any other where conditions, just symbol=my symbol.  Most
        // of the where conditions can be disabled in the GUI/API.  But you
        // have to supply a sort condition.  You have to supply that even
        // though you're only asking for one record, so sorting doesn't do
        // anything.  What's the best sort field to request in this case?  It
        // turns out that almost any standard filter can be NULL sometimes.  So
        // there's nothing good that the client can put into the sort field to
        // ensure he get's the data he's asking for.
        //
        // At this time this rule only applies to realtime top lists.  I
        // haven't moved this to the database code.  I'm not sure if I ever
        // will.  This is such a specific rule, and over time more programs
        // will start to use this file, rather than the old database code.
        // 3/11/2016
        resultSoFar.push_back(new ValidDouble(getSortField()));
      for (std::string const &name : getFilterNames())
      {
        double const *minValue =
          getProperty(getFilterList().getMin(), name);
        double const *maxValue =
          getProperty(getFilterList().getMax(), name);
        if (minValue)
        {
          if (maxValue)
            resultSoFar.push_back(bothFilters(getFilterValue(name),
                                              *minValue, *maxValue));
          else
            resultSoFar.push_back(minFilter(getFilterValue(name), *minValue));
        }
        else
        {
          if (maxValue)
            resultSoFar.push_back(maxFilter(getFilterValue(name), *maxValue));
        }
      }
      return new AndFunction(resultSoFar);
    }
    virtual BitSet const &getActiveExchanges() const
    { return getExchanges().getActiveExchanges(); }
    virtual SymbolLists const &getSymbolLists() const
    { return TopListConfig::getSymbolLists(); }
    virtual ColumnListConfig const &getColumnListConfig() const
    { return getColumnList(); }
  public:
    NewTopListConfig(DatabaseWithRetry &database, UserId userId) :
      NewConfigBase(database, userId), _addMagicColumns(false) { }

    void setAddMagicColumns(bool value) { _addMagicColumns = value; }
    bool getAddMagicColumns() const { return _addMagicColumns; }

    virtual Tree getSort() const
    {
      if (getSortBy().highestOnTop())
        return new MinusFunction(Token::EMPTY, getSortField());
      else
        return getSortField();
    }
    virtual ColumnsByName getColumns() const
    {
      ColumnsByName result = NewConfigBase::getColumns();
      if (_addMagicColumns)
      {
        result["EXCHANGE"] = AlertsDaily::LIST_EXCH;
        result["symbol"] = PrimaryField::SYMBOL;
      }
      return result;
    }
    int getCount() const { return TopListConfig::getCount(); }
    bool outsideMarketHours() const
    { return getTime().outsideMarketHours(); }
    std::string getSingleSymbol() const
    { return getSymbolLists().getSingleSymbol(); }
    bool hasSingleSymbol() const { return !getSingleSymbol().empty(); }

    virtual StrategyTrees getStrategyTrees() const
    { return NewConfigBase::getStrategyTrees(); }

    void getInitialDescription(XmlNode &node) const
    {
      TopListConfig::getInitialDescription(node,
                                           getUserId(),
                                           *DatabaseForThread(DatabaseWithRetry::SLAVE));
    }
  };


  /////////////////////////////////////////////////////////////////////
  // StrategyTrees
  /////////////////////////////////////////////////////////////////////

  Tree StrategyTrees::usePrice(bool current, Tree orig)
  {
    if (current)
      return AppropriatePrice::useCurrentPrice(orig);
    else
      return AppropriatePrice::useLastPrice(orig);
  }

  void StrategyTrees::usePrice(bool current)
  {
    assert(_optimized == OPT_NONE);
    _where = usePrice(current, _where);
    _sort = usePrice(current, _sort);
    for (auto &kvp : _columns)
      kvp.second = usePrice(current, kvp.second);
  }

  void StrategyTrees::useCurrentPrice()
  {
    usePrice(true);
  }

  void StrategyTrees::useLastPrice()
  {
    usePrice(false);
  }

  void StrategyTrees::countDuplicates(Tree tree, Counts &counts)
  {
    if (tree->easyToCompute())
      return;
    Count &count = counts[tree];
    if (count == count0)
    {
      count = count1;
      for (Tree const &child : tree->getChildren())
        countDuplicates(child, counts);
    }
    else
      count = countMore;
  }

  // You can't use the normal replace() here.  That starts at the bottom of
  // the tree and works it's way up.  If you could have replaced this whole
  // tree, or one of it's sub-trees, you want the whole tree.  If you replace
  // the sub-tree first, then the top level tree won't match any more.
  Tree StrategyTrees::replaceTopDown(Tree original,
                                     Replacements const &replacements)
  {
    Replacements::const_iterator it = replacements.find(original);
    if (it != replacements.end())
      // Found an exact match for the entire tree.
      return it->second;
    bool somethingChanged = false;
    TreeNode::Args children = original->getChildren();
    for (Tree &child : children)
    {
      Tree newValue = replaceTopDown(child, replacements);
      if (newValue->compare(*child))
      {
        child = newValue;
        somethingChanged = true;
      }
    }
    if (somethingChanged)
      return original->construct(children);
    else
      return original;
  }

  static const std::string s_optimizePartial = "optimizePartial()";
  void StrategyTrees::optimizePartial()
  {
    if (_optimized == OPT_FULL)
      throw std::logic_error("already fully optimized");
    if (_optimized == OPT_PARTIAL)
      return;
    // We only want to know the number of times that we did work.
    ThreadMonitor::SetState tm(s_optimizePartial);
    tm.increment(s_optimizePartial);
    _where = TreeNode::optimize(_where, OptimizationContext::FOR_BOOLEAN);
    _sort = TreeNode::optimize(_sort);
    for (auto &kvp : _columns)
      kvp.second = TreeNode::optimize(kvp.second);
    _optimized = OPT_PARTIAL;
  }

  static const std::string s_optimizeFull = "optimizeFull()";
  void StrategyTrees::optimizeFull(std::vector< StrategyTrees * > const &trees)
  {
    ThreadMonitor::SetState tm(s_optimizeFull);
    tm.increment(s_optimizeFull);
    // OptimizePartial() will record the number of individual strategies
    // optimized, trees.size(), so no need to repeat it here.

    for (StrategyTrees *item : trees)
    {
      item->optimizePartial();  // This will assert item->_optimized!=OPT_FULL
      item->_optimized = OPT_FULL;  // So we can only do this once!
    }

    Counts counts;
    for (StrategyTrees *const item : trees)
    {
      countDuplicates(item->_where, counts);
      countDuplicates(item->_sort, counts);
      for (auto const &kvp : item->_columns)
        countDuplicates(kvp.second, counts);
    }

    // We know which trees are reused.  Now sort those trees by height, with
    // the shortest trees first.  If the tree A contains the tree B, we know
    // that we'll find B before A in our list.  It's possible for one cached
    // item to contain another.  In fact, we expect that to happen a lot.
    typedef std::set< std::pair< int, Tree > > SortedByHeight;
    SortedByHeight sortedByHeight;
    for (auto const &kvp : counts)
    {
      const Count count = kvp.second;
      assert(count != count0);
      if (count == countMore)
        sortedByHeight.insert(std::pair< int, Tree >(kvp.first->height(),
                                                     kvp.first));
    }
    Replacements replacements;
    for (auto const &kvp : sortedByHeight)
    {
      const Tree value = kvp.second;
      const Tree replacement =
        new CachedValue(replacements.size(),
                        replaceTopDown(value, replacements));
      replacements[value] = replacement;
    }

    for (auto it = trees.cbegin(); it != trees.cend(); it++)
    {
      StrategyTrees *item = *it;
      item->_where = replaceTopDown(item->_where, replacements);
      item->_sort = replaceTopDown(item->_sort, replacements);
      for (ColumnsByName::iterator it = item->_columns.begin();
           it != item->_columns.end();
           it++)
        it->second = replaceTopDown(it->second, replacements);
      item->_cacheSize = replacements.size();
    }
  }

  std::string StrategyTrees::shortDebug() const
  {
    CachedValue::startDebug();
    TclList result;
    result<<"WHERE"<<_where->shortDebug();
    result<<"SORT"<<_sort->shortDebug();
    int i = 0;
    for (ColumnsByName::const_iterator it = _columns.begin();
         it != _columns.end();
         it++, i++)
      result<<"COLUMN"<<it->first<<(it->second)->shortDebug();
    CachedValue::endDebug();
    return result;
  }

  bool StrategyTrees::operator ==(StrategyTrees const &other) const
  {
    if (_where->compare(*other._where))
      return false;
    if (_sort->compare(*other._sort))
      return false;
    if (_columns.size() != other._columns.size())
      return false;
    for (ColumnsByName::const_iterator it = _columns.begin();
         it != _columns.end();
         it++)
    {
      Tree otherValue = getPropertyDefault(other._columns, it->first);
      if (!otherValue)
        // Key is in this tree, but not the other tree.
        return false;
      if ((it->second)->compare(*otherValue))
        // Same key, different value.
        return false;
    }
    // What about _cacheSize and _optimized()?
    return true;
  }


  /////////////////////////////////////////////////////////////////////
  // CompiledStrategyTrees
  /////////////////////////////////////////////////////////////////////

  Execution::Executable *CompiledStrategyTrees::empty()
  {
    static const Tree TREE = new NullValue(Token::EMPTY);
    return Execution::Executable::create(TREE);
  }

  Execution::Executable *CompiledStrategyTrees::create(Tree tree)
  {
    if (!tree)
      // This is not unreasonable.  In particular, sort is null when we are
      // in an alert window.
      return empty();
    try
    {
      return Execution::Executable::create(tree);
    }
    catch (Exception const &ex)
    { // This shouldn't happen in production code.  It can happen in
      // development, and often does.  We could pass this exception on, but
      // it's a pain to handle exceptions in a constructor, so this is as
      // good a place as any to catch this.
      sendToLogFile(TclList()<<FLF
                    <<ex.getMessage()<<ex.simpleLocationMessage());
      return empty();
    }
  }

  CompiledStrategyTrees::CompiledStrategyTrees(StrategyTrees const &trees)
  {
    _where = create(trees.getWhere());
    _sort = create(trees.getSort());
    for (ColumnsByName::const_iterator it = trees.getColumns().begin();
         it != trees.getColumns().end();
         it++)
      _columns[it->first] = create(it->second);
  }

  CompiledStrategyTrees::~CompiledStrategyTrees()
  {
    delete _where;
    delete _sort;
    for (Columns::const_iterator it = _columns.begin();
         it != _columns.end();
         it++)
      delete it->second;
  }

  static const std::string s_evaluateWhere = "evaluateWhere";

  bool CompiledStrategyTrees::evaluateWhere(Execution::RecordInfo &recordInfo
                                            ) const
  {
    return _where->execute(recordInfo).getBoolean();
  }

  static const std::string s_evaluateSort = "evaluateSort";

  double CompiledStrategyTrees::evaluateSort(Execution::RecordInfo &recordInfo
                                             ) const
  {
    ValueBox value = _sort->execute(recordInfo);
    bool valid;
    double asDouble;
    value.getDouble(valid, asDouble);
    if (valid)
      return asDouble;
    else
      // This shouldn't happen, but I'm not sure that I want to fail an
      // assertion.
      // Originally this would return NaN.  But NaN doesn't sort well.
      // (A a key in a std::map it can actually cause core dumps!)  Negative
      // infinity should always be at the bottom of the list.
      return -std::numeric_limits< double >::infinity();
  }

  static const std::string s_evaluateColumns = "evaluateColumns";

  ValuesByName CompiledStrategyTrees::evaluateColumns
  (Execution::RecordInfo &recordInfo) const
  {
    ValuesByName result;
    for (Columns::const_iterator it = _columns.begin();
         it != _columns.end();
         it++)
    {
      ValueBox value = it->second->execute(recordInfo);
      if (!value.isEmpty())
        result[it->first] = value;
    }
    return result;
  }

  void CompiledStrategyTrees::cacheInfoDebug(std::map< int, int > &used) const
  {
    _where->cacheInfoDebug(used);
    if (_sort)
      _sort->cacheInfoDebug(used);
    for (auto it = _columns.cbegin(); it != _columns.cend(); it++)
      it->second->cacheInfoDebug(used);
  }

  std::string CompiledStrategyTrees::describeDebug() const
  {
    Execution::Executable::TransactionDebug transaction;
    TclList result;
    result<<"_where";
    if (_where)
      result<<_where->nameDebug();
    else
      result<<"NULL";
    result<<"_sort";
    if (_sort)
      result<<_sort->nameDebug();
    else
      result<<"NULL";
    result<<"_columns";
    TclList columns;
    for (auto it = _columns.begin(); it != _columns.end(); it++)
    {
      columns<<it->first;
      if (it->second)
        columns<<it->second->nameDebug();
      else
        columns<<"NULL";
    }
    result<<columns;
    return result;
  }


  /////////////////////////////////////////////////////////////////////
  // AlertStrategy
  /////////////////////////////////////////////////////////////////////

  AlertStrategy::AlertStrategy() : _cacheSize(0) { }

  void AlertStrategy::load(std::string const &collaborate, UserId userId,
                           DatabaseWithRetry &database,
                           bool compileNow, TclList *debugDump)
  {
    if (debugDump)
      (*debugDump)<<FLF
                  <<"collaborate"<<collaborate
                  <<"userId"<<userId;

    NewAlertConfig config(database, userId);
    config.load(collaborate, userId, database, true, true);
    config.removeIllegalData(userId, database);
    _halfOptimized = config.getStrategyTrees();
    _halfOptimized.useCurrentPrice();
    if (debugDump)
      (*debugDump)<<"unoptimized"<<_halfOptimized.shortDebug();
    _halfOptimized.optimizePartial();
    if (debugDump)
      (*debugDump)<<"partially optimized"<<_halfOptimized.shortDebug();
    if (compileNow)
      compile();
    else
    {
      _compiled = NULL;
      _cacheSize = 0;
    }
  }

  static const std::string s_AlertStrategy_compile = "AlertStrategy::compile";
  static const std::string s_AlertStrategy_compile_test =
                                           "AlertStrategy::compile test";

  void
  AlertStrategy::compileImpl(std::vector< AlertStrategy * > const &strategies)
  {
    // Make a copy of the trees.  We don't want to optimize the original.  We
    // couldn't undo that.  We need the original in case we decide to recompile
    // this strategy as part of a different set of strategies.
    std::vector< StrategyTrees > allTrees;
    allTrees.reserve(strategies.size());
    std::vector< StrategyTrees * > allTreePtrs;
    allTreePtrs.reserve(strategies.size());
    for (auto it = strategies.begin(); it != strategies.end(); it++)
      allTrees.push_back((*it)->_halfOptimized);
    for (auto it = allTrees.begin(); it != allTrees.end(); it++)
      allTreePtrs.push_back(&*it);
    StrategyTrees::optimizeFull(allTreePtrs);
    auto treesIt = allTrees.cbegin();
    for (auto strategiesIt = strategies.cbegin();
         strategiesIt != strategies.cend();
         strategiesIt++, treesIt++)
    {
      (*strategiesIt)->_compiled.emplace(*treesIt);
      (*strategiesIt)->_cacheSize = treesIt->getCacheSize();
    }
    assert(treesIt == allTrees.cend());
  }

  TclList AlertStrategy::compileVerify(std::vector< AlertStrategy * > const &
                                       strategies)
  {
    //static int debugCount = 0;
    //debugCount++;
    //if ((debugCount%10)==0)
    //  return TclList()<<(TclList()<<"fake failure");

    TclList errorMessage;
    if (!strategies.empty())
    {
      const auto size = strategies[0]->_cacheSize;
      std::map< int, int > cacheUse;
      for (auto it = strategies.begin(); it != strategies.end(); it++)
      {
        if (size != (*it)->_cacheSize)
          errorMessage<<(TclList()<<"_cacheSize"<<(*it)->_cacheSize
                         <<"expecting"<<size);
        (*it)->_compiled->cacheInfoDebug(cacheUse);
      }
      // New rules.  We were reporting any time the cache didn't look exactly
      // as we expected.  That caused huge log files.  This is still
      // suspicious, but not that important right now.  For now I'm ignorning
      // cache entries that are reserved but never used.  For now I'm only
      // looking for cache entries that are used but not reserved.  I haven't
      // seen any like that, yet.  If I ever do, that would explain some
      // core dumps.
      if (!cacheUse.empty())
      { // First make sure that the keys will fit into the cache vector.
        // Nothing out of range.
        if (cacheUse.begin()->first < 0)
          errorMessage<<(TclList()
                         <<"first cache entry"<<cacheUse.rbegin()->first
                         <<"expecting"<<"0");
        if (cacheUse.rbegin()->first > size - 1)
          errorMessage<<(TclList()
                         <<"last cache entry"<<cacheUse.rbegin()->first
                         <<"expecting"<<(size-1));
        // And no empty space.  That could only be caused by some sort of bug.
        //if (size != (int)cacheUse.size())
        //  errorMessage<<(TclList()
        //               <<"empty space in cache"<<(size - cacheUse.size()));
        for (auto it = cacheUse.begin(); it != cacheUse.end(); it++)
          // Each cache entry should be used at least two places.  If it's only
          // used in one place, why did we bother to reserve space for it?
          if (it->second <= 1)
            errorMessage<<(TclList()<<"invalid cache use count"<<it->second);
      }
    }
    return errorMessage;
  }

  void AlertStrategy::dumpCacheToLog(std::vector< AlertStrategy * > const &
                                     strategies)
  {
    std::map< int, int > cacheUse;
    CachedValue::startDebug();  // For _halfOptimized.
    Execution::Executable::TransactionDebug transaction;  // For _compiled.
    for (auto it = strategies.cbegin(); it != strategies.cend(); it++)
    {
      TclList msg;
      msg<<FLF<<"strategy"
         <<(*it)->_halfOptimized.shortDebug()
         <<(*it)->_compiled->describeDebug();
      sendToLogFile(msg);
      (*it)->_compiled->cacheInfoDebug(cacheUse);
    }
    CachedValue::endDebug();
    TclList msg;
    msg<<FLF<<"cacheUse"<<cacheUse<<"strategies.size()"<<strategies.size();
    if (!strategies.empty())
      msg<<"_cacheSize"<<strategies[0]->_cacheSize;
    sendToLogFile(msg);
  }

  void AlertStrategy::compile(std::vector< AlertStrategy * > const &strategies)
  {
    ThreadMonitor::SetState tm(s_AlertStrategy_compile);
    compileImpl(strategies);
    tm.setState(s_AlertStrategy_compile_test);
    TclList status = compileVerify(strategies);
    if (status.empty())
      return;
    sendToLogFile(TclList()<<FLF<<"COMPILE CACHE PROBLEM"<<status);
    dumpCacheToLog(strategies);
    tm.setState(s_AlertStrategy_compile);
    compileImpl(strategies);
    tm.setState(s_AlertStrategy_compile_test);
    status = compileVerify(strategies);
    if (status.empty())
      // Success the second time, even though we didn't change anything.
      return;
    // Failed again.  Presumably the exact same input caused the exact same
    // problem.  But check the logs to be sure.
    sendToLogFile(TclList()<<FLF<<"COMPILE CACHE PROBLEM"<<"DUP"<<status);
  }

  void AlertStrategy::compile()
  {
    StrategyTrees temp = _halfOptimized;
    temp.optimizeFull();
    _compiled.emplace(temp);
    _cacheSize = temp.getCacheSize();
  }

  static void initCache(Execution::RecordInfo &recordInfo, int cacheSize)
  {
    recordInfo.cache.clear();
    if (cacheSize > 0)
    {
      ValueBox v;
      v.setCacheEmpty();
      recordInfo.cache.resize(cacheSize, v);
    }
  }

  void AlertStrategy::init(Execution::RecordInfo &recordInfo) const
  {
    initCache(recordInfo, _cacheSize);
  }

  bool AlertStrategy::evaluateWhere(Execution::RecordInfo &recordInfo) const
  {
    return _compiled->evaluateWhere(recordInfo);
  }

  ValuesByName
  AlertStrategy::evaluateColumns(Execution::RecordInfo &recordInfo) const
  {
    return _compiled->evaluateColumns(recordInfo);
  }


  /////////////////////////////////////////////////////////////////////
  // TopListStrategyBase::Result
  /////////////////////////////////////////////////////////////////////

  void TopListStrategyBase::Result::addAsChild(XmlNode &parent,
                                               std::string const &windowId
                                               ) const
  {
    XmlNode &body = parent["TOPLIST"];
    body.properties["START_TIME"] = ntoa(start);
    body.properties["END_TIME"] = ntoa(end);
    body.properties["TYPE"] = "data";
    body.properties["WINDOW"] = windowId;
    for (Values::const_iterator rowIt = values.begin();
         rowIt != values.end();
         rowIt++)
    {
      XmlNode &row = body[-1];
      for (ValuesByName::const_iterator columnIt = rowIt->begin();
           columnIt != rowIt->end();
           columnIt++)
        // Write this name-value pair to the XML document.
        columnIt->second.writeToClient(row.properties, columnIt->first);
    }
  }

  TopListStrategyBase::Result::Result(Result &&other) :
    values(std::move(other.values)), start(other.start), end(other.end) { }

  void TopListStrategyBase::Result::operator =(Result const &other)
  {
    values = other.values;
    start = other.start;
    end = other.end;
  }

  void TopListStrategyBase::Result::operator =(Result && other)
  {
    values = std::move(other.values);
    start = other.start;
    end = other.end;
  }


  /////////////////////////////////////////////////////////////////////
  // TopListStrategyBase
  /////////////////////////////////////////////////////////////////////

  TopListStrategyBase::TopListStrategyBase() : _resultCount(0) { }

  // Runs a list of strategies.  Requests and results are both stored in
  // the same data structure.  This directly fills in the data shown in the
  // top list grid.  Time is filled in elsewhere.
  void TopListStrategyBase::run(Records const &records,
                                StrategiesInProgress &strategies,
                                int cacheSize)
  {
    // First look at requests with a single symbol.  We can jump directly to
    // that symbol and not walk through the entire table.
    std::vector< int > fullTableScan;
    for (size_t i = 0; i < strategies.size(); i++)
    {
      StrategyInProgress &strategy = strategies[i];

      // Seems like clear() should be called much sooner.  When someone takes
      // the results, it should be a move, not a copy.  That's faster, but it
      // also means we don't keep a copy of the old results in memory as part
      // of the strategy list.
      strategy.result.values.clear();

      if (strategy.strategy->hasSingleSymbol())
      { // Do it the easy way!
        ThreadMonitor::SetState tm("TopListStrategyBase::run easy");
        tm.increment("TopListStrategyBase::run easy");
        // The next line is making a pointer to a smart pointer.  In this
        // case perhaps it's overkill.  Execution::RecordInfo explicitly tries
        // to avoid making a copy of a smart pointer.  Since we're only looking
        // at one record here, it's probably not that important.  It did
        // make a measurable difference when dealing with large numbers of
        // alert or top list records.  The next line is as efficient as the
        // the case when we have lots of records mostly for consistency.
        Record::Ref const *record =
          getProperty(records, strategy.strategy->getSingleSymbol());
        if (!record)
          tm.increment("TopListStrategyBase::run symbol not found");
        else
        {
          Execution::RecordInfo recordInfo;
          recordInfo.setRecord(*record);
          initCache(recordInfo, cacheSize);
          if (strategy.executables->evaluateWhere(recordInfo))
            strategy.result.values.push_back(strategy.executables
                                             ->evaluateColumns(recordInfo));
        }
      }
      else if (strategy.executables)
        // Do it soon, possibly grouped with others that also need a full table
        // scan.
        fullTableScan.push_back(i);
      else
        // No work to do at all for this strategy.
        ThreadMonitor::find().increment("TopListStrategyBase::run no work");
    }

    if (!fullTableScan.empty())
    {
      ThreadMonitor::SetState tm("TopListStrategyBase::run fullTableScan");
      tm.increment("TopListStrategyBase::run fullTableScan setup");
      tm.increment("TopListStrategyBase::run fullTableScan strategy",
                   fullTableScan.size());
      typedef std::vector< Execution::RecordInfo > AllRecordInfo;
      AllRecordInfo allRecordInfo;
      allRecordInfo.resize(records.size());
      {
        int i = 0;
        for (auto it = records.cbegin(); it != records.cend(); it++, i++)
        {
          Execution::RecordInfo &recordInfo = allRecordInfo[i];
          initCache(recordInfo, cacheSize);
          recordInfo.setRecord(it->second);
        }
      }
      typedef std::vector< PossibleResult > PossibleResults;
      typedef std::vector< PossibleResults > AllPossibleResults;
      AllPossibleResults allPossibleResults;
      allPossibleResults.resize(strategies.size());
      for (auto it = allRecordInfo.begin(); it != allRecordInfo.end(); it++)
      {
        for (size_t i = 0; i < strategies.size(); i++)
        {
          StrategyInProgress const &strategy = strategies[i];
          if (strategy.executables)
          {
            PossibleResult r;
            r.recordInfo = &*it;

            /*
            // TEST ONLY.  This will print a lot of details to the standard
            // output each time we try to compare Microsoft's current data
            // to the user's top list strategy.  This is also a useful place
            // to set a breakpoint.  Microsoft is typical of the stocks that
            // we care about.  If you set a breakpoint on the line above you'll
            // have to wade through a lot of stocks that barely trade and are
            // missing a lot of data.
            if (Execution::getSymbol(r.recordInfo->currentRecord) == "MSFT")
            {
            puts(Execution::debugDump(r.recordInfo->currentRecord).c_str());
            puts(strategy.executables->describeDebug().c_str());
            }
            */

            if (strategy.executables->evaluateWhere(*r.recordInfo))
            {
              r.sortOn = strategy.executables->evaluateSort(*r.recordInfo);
              if (finite(r.sortOn))  // This should always be true!
                allPossibleResults[i].push_back(r);
            }
          }
        }
      }

      for (size_t i = 0; i < strategies.size(); i++)
      {
        PossibleResults &possibleResults = allPossibleResults[i];
        StrategyInProgress &strategy = strategies[i];
        std::sort(possibleResults.begin(), possibleResults.end());
        int count = std::min((int)possibleResults.size(),
                             strategy.strategy->getResultCount());
        for (PossibleResults::iterator it = possibleResults.begin();
             count > 0;
             count--, it++)
          strategy.result.values.push_back(strategy.executables
                                           ->evaluateColumns(*it->recordInfo));
      }
    }
  }

  // Runs a list of strategies.  Requests and results are both stored in
  // the same data structure.  Directly takes care of time.  Calls another
  // function to fill in the bulk of the data.
  void TopListStrategyBase::run(Records const &records,
                                StrategiesInProgress &strategies,
                                int cacheSize, time_t endTime, int seconds)
  {
    run(records, strategies, cacheSize);
    for (auto it = strategies.begin(); it != strategies.end(); it++)
      if (!it->executables)
      {
        it->result.start = 0;
        it->result.end = 0;
      }
      else
      {
        it->result.start = endTime - seconds + 1;
        it->result.end = endTime;
      }
  }


  /////////////////////////////////////////////////////////////////////
  // TopListStrategy
  /////////////////////////////////////////////////////////////////////

  TopListStrategy::Result TopListStrategy::run(Records const &records,
                                               time_t endTime,
                                               int seconds) const
  { // Assume the market is open.
    StrategiesInProgress strategies;
    strategies.emplace_back(this, _streaming);
    TopListStrategyBase::run(records, strategies, getCacheSize(),
                             endTime, seconds);
    return strategies[0].result;
  }

  TopListStrategy::Result TopListStrategy::run(Records const &records,
                                               time_t endTime,
                                               Records const &frozenRecords,
                                               time_t frozenEndTime,
                                               int seconds) const
  { // Assume the market is closed.
    StrategiesInProgress strategies;
    if (_frozen)
    { // Look at the data which was frozen at the close.
      strategies.emplace_back(this, _frozen);
      TopListStrategyBase::run(frozenRecords, strategies, getCacheSize(),
                               frozenEndTime, seconds);
    }
    else
    {
      strategies.emplace_back(this, _streaming);
      TopListStrategyBase::run(records, strategies, getCacheSize(),
                               endTime, seconds);
    }
    return strategies[0].result;
  }

  TopListStrategy::TopListStrategy() : _cacheSize(0) { }

  ITopListConfigFromCollaborate *
  TopListStrategy::getConfig(std::string collaborate,
                             UserId userId,
                             DatabaseWithRetry &database,
                             bool saveToMru, bool magicColumns,
                             bool strategyColumns,
                             TclList *debugDump)
  {
    if (debugDump)
      (*debugDump)<<FLF
                  <<"collaborate"<<collaborate
                  <<"userId"<<userId;
    NewTopListConfig *config = new NewTopListConfig(database, userId);
    config->load(collaborate, userId, database, true);
    config->removeIllegalData(userId, database);
    config->setAddMagicColumns(magicColumns);

    // TODO use saveToMru,
    // TODO Move strategyColumns somewhere else.
    return config;
  }

  void TopListStrategy::load(std::string collaborate, UserId userId,
                             DatabaseWithRetry &database, TclList *debugDump)
  {
    ITopListConfig *config =
      getConfig(collaborate, userId, database,
                /* save to MRU */ false,
                /* magicColumns */ true,
                /* strategyColumns */ true,
                debugDump);
    if (debugDump)
      (*debugDump)<<FLF
                  <<"collaborate"<<collaborate
                  <<"userId"<<userId;
    load(*config, debugDump);
    delete config;
  }

  void TopListStrategy::load(ITopListConfig &config, TclList *debugDump)
  {
    _resultCount = config.getCount();
    _singleSymbol = config.getSingleSymbol();
    const StrategyTrees baseTrees = config.getStrategyTrees();
    StrategyTrees liveTrees = baseTrees;
    liveTrees.useCurrentPrice();
    if (debugDump)
      (*debugDump)<<"liveTrees unoptimized"<<liveTrees.shortDebug();
    liveTrees.optimizeFull();
    if (debugDump)
      (*debugDump)<<"liveTrees"<<liveTrees.shortDebug();
    _streaming.emplace(liveTrees);
    if (config.outsideMarketHours())
    { // Do not do anything special outside market hours.  Handle these
      // requests the same way during market hours.
      _cacheSize = liveTrees.getCacheSize();
      _frozen = NULL;
    }
    else
    { // Freeze the data at the close.  After market hours use that data,
      // and slightly modified formulae.
      StrategyTrees frozenTrees = baseTrees;
      frozenTrees.useLastPrice();
      frozenTrees.optimizeFull();
      if (debugDump)
        (*debugDump)<<"frozenTrees"<<frozenTrees.shortDebug();
      _frozen.emplace(frozenTrees);
      // For the sake of the cache, we assume that the caller will consistently
      // apply one strategy or the other to a RecordInfo.  We don't know which
      // one when init() is called, but we know that it never changes.
      _cacheSize =
        std::max(liveTrees.getCacheSize(), frozenTrees.getCacheSize());
    }
  }

  void TopListStrategy::init(Execution::RecordInfo &recordInfo) const
  {
    initCache(recordInfo, _cacheSize);
  }


  /////////////////////////////////////////////////////////////////////
  // MergableTopListStrategy::List
  /////////////////////////////////////////////////////////////////////

  TopListStrategyBase::StrategiesInProgress &
  MergableTopListStrategy::List::getStrategies(bool live, int &cacheSize)
  {
    StrategiesInProgress &storedStrategies =
      live?_liveStrategies:_frozenStrategies;
    int &storedCacheSize = live?_liveCacheSize:_frozenCacheSize;
    if (storedStrategies.size() < _strategies.size())
    { // Initialize it.
      std::vector< StrategyTrees > allTrees;
      for (auto it = _strategies.begin(); it != _strategies.end(); it++)
        if (live)
          allTrees.push_back(it->_liveTrees);
        else
          allTrees.push_back(it->_frozenTrees);
      std::vector< StrategyTrees * > allTreePtrs;
      for (auto it = allTrees.begin(); it != allTrees.end(); it++)
        allTreePtrs.push_back(&*it);
      StrategyTrees::optimizeFull(allTreePtrs);
      for (size_t i = 0; i < _strategies.size(); i++)
        storedStrategies.emplace_back(&_strategies[i],
                                      CompiledStrategyTrees::Ref(NULL,
                                                                 allTrees[i]));
      if (allTrees.empty())
        storedCacheSize = 0;
      else
        storedCacheSize = allTrees[0].getCacheSize();
    }
    cacheSize = storedCacheSize;
    return storedStrategies;
  }

  MergableTopListStrategy::List::List(std::vector< MergableTopListStrategy >
                                      const &strategies) :
    _strategies(strategies), _liveCacheSize(0), _frozenCacheSize(0) { }

  // A list of strategies returns a list of results.  Assumes the market is
  // open.  Called from outside this class.
  std::vector< TopListStrategyBase::Result >
  MergableTopListStrategy::List::run(Records const &records,
                                     time_t endTime, int seconds)
  {
    int cacheSize;
    StrategiesInProgress &inProgress = getStrategies(true, cacheSize);
    TopListStrategyBase::run(records, inProgress, cacheSize, endTime, seconds);
    std::vector< Result > result;
    result.reserve(inProgress.size());
    for (auto it = inProgress.begin(); it != inProgress.end(); it++)
      result.emplace_back(std::move(it->result));
    return result;
  }


  /////////////////////////////////////////////////////////////////////
  // MergableTopListStrategy
  /////////////////////////////////////////////////////////////////////

  void MergableTopListStrategy::load(std::string collaborate, UserId userId,
                                     DatabaseWithRetry &database)
  {
    NewTopListConfig config(database, userId);
    config.load(collaborate, userId, database, true);
    config.removeIllegalData(userId, database);
    _resultCount = config.getCount();
    _singleSymbol = config.getSingleSymbol();
    _liveTrees = config.getStrategyTrees();
    _frozenTrees = _liveTrees;
    _liveTrees.useCurrentPrice();
    _liveTrees.optimizePartial();
    if (config.outsideMarketHours())
    { // Do not do anything special outside market hours.  Handle these
      // requests the same way during market hours.
      _frozenTrees = _liveTrees;
    }
    else
    {
      // Freeze the data at the close.  After market hours use that data,
      // and slightly modified formulae.
      _frozenTrees.useLastPrice();
      _frozenTrees.optimizePartial();
    }
  }

};

/////////////////////////////////////////////////////////////////////
// Unit Test
/////////////////////////////////////////////////////////////////////

#ifdef UNIT_TEST_STRATEGY

// g++ -L/usr/lib64/mysql -lmysqlclient_r -lpcre -DUNIT_TEST_STRATEGY -std=c++0x -o unit_test_strategy -ggdb -Wall -I../stocktwits/json -DJSON_ISO_STRICT Strategy.C AlertsDailyData.C AlertsDailyDatabase.C Execution.C Parse.C Semantics.C DataFormat.C FieldLists.C IAlertsDaily.C ../ax_alert_server/BitSet.C ../ax_alert_server/AlertConfig.C ../ax_alert_server/MiscSQL.C ../shared/GlobalConfigFile.C ../shared/DatabaseWithRetry.C ../shared/MiscSupport.C ../shared/FixedMalloc.C ../shared/ThreadMonitor.C ../shared/SimpleLogFile.C -Wno-reorder ../stocktwits/json/Source/*.cpp

// Take a look at SimpleAlertTest.C.  That's probably a better way to test
// this unit.

#include <iostream>

#include "../shared/GlobalConfigFile.h"
#include "AlertsDailyDatabase.h"

static void help()
{
  std::cout<<"alert_id="<<std::endl
           <<"user_id="<<std::endl
           <<"date="<<std::endl
           <<"collaborate="<<std::endl;
}

int main(int args, char **argv)
{
  setlocale(LC_ALL, "");

  addConfigItems("database_RO=wally-balls");
  addConfigItems("log_file_suffix=TestFromDatabase");
  if (!addConfigItemsFromCommandLine(argv + 1))
  {
    return 1;
  }
  configItemsComplete();
  AllConfigInfo::init();
  DatabaseWithRetry database(true, "ro database");
  UserId userId = 0;
  std::string collaborate;
  Parse::StrategyTrees asTrees;
  Parse::AlertStrategy *alertStrategy = NULL;
  AlertsDailyData alertsDailyData;
  alertsDailyData.makePrimary();
  Record::Ref record;
  while (std::cin)
  {
    bool recompile = false;
    bool execute = false;
    std::cout<<"? ";
    std::string line;
    getline(std::cin, line);
    if (!std::cin)
      return 0;
    std::string command = breakOnce(line, '=');
    std::vector< DatabaseFieldInfo > const &fields =
      DatabaseFieldInfo::getAlertFields();
    if (command == "alert_id")
    {
      std::string sql = "SELECT "
        + DatabaseFieldInfo::makeSqlFieldList(fields)
        + " FROM alerts WHERE id='" + mysqlEscapeString(line) + "'";
      MysqlResultRef result = database.tryQueryUntilSuccess(sql);
      if (result->rowIsValid())
      {
        record = DatabaseFieldInfo::makeRecord(result, fields);
        std::cout<<DatabaseFieldInfo::debugDump(record, fields)<<std::endl;
        execute = true;
      }
      else
      {
        record = NULL;
        std::cout<<"No record found."<<std::endl;
      }
    }
    else if (command == "user_id")
    {
      userId = strtolDefault(line, 0);
      recompile = true;
    }
    else if (command == "date")
    {
      int count = add(alertsDailyData, line, database);
      std::cout<<"Loaded "<<ntoa(count)<<" records from alerts daily."
               <<std::endl;
      execute = true;
    }
    else if (command == "collaborate")
    {
      collaborate = line;
      recompile = true;
    }
    else
      help();

    if (recompile)
    {
      if (collaborate.empty())
      {
        delete alertStrategy;
        alertStrategy = NULL;
      }
      else
      {
        Parse::StrategyTrees newTrees =
          Parse::AlertStrategy::parse(collaborate, userId, database);
        Parse::StrategyTrees optimized = newTrees;
        optimized.optimize();
        if (!(asTrees == optimized))
        {
          std::cout<<"Unoptimized:  "<<std::endl;
          std::cout<<newTrees.shortDebug()<<std::endl;
          std::cout<<"Optimized:  "<<std::endl;
          Parse::CachedValue::startDebug();
          std::cout<<optimized.shortDebug()<<std::endl;
          Parse::CachedValue::endDebug();
          asTrees = optimized;
          execute = true;
          delete alertStrategy;
          alertStrategy = new Parse::AlertStrategy(asTrees, userId, database);
        }
      }
    }

    if (execute && alertStrategy && record)
    {
      Execution::RecordInfo recordInfo;
      recordInfo.currentRecord = record;
      alertStrategy->init(recordInfo);
      std::cout<<"where: "
               <<(alertStrategy->evaluateWhere(recordInfo)?"TRUE":"FALSE")
               <<std::endl;
      const std::vector< Value::Ref > columns =
        alertStrategy->evaluateColumns(recordInfo);
      for (int i = 0; i < (int)columns.size(); i++)
        std::cout<<"column "<<ntoa(i)<<":  "
                 <<columns[i]->shortDebug()<<std::endl;
    }

  }
}

#endif