#include <cmath>
#include <limits>
#include <algorithm>
#include <fstream>

#include "../misc_framework/CsvFileDataNodes.h"
#include "../misc_framework/GenericDataNodes.h"
#include "../data_framework/GenericTosData.h"
#include "../data_framework/StandardCandles.h"
#include "../data_framework/SynchronizedTimers.h"
#include "../misc_framework/Timers.h"
#include "../misc_framework/StandardPlaceholders.h"
#include "TiqMisc.h"
#include "SmbLR.h"
#include "ReportAlertsThread.h"

/* Here are the initial requirements for the SMB Radar:
 * https://docs.google.com/a/trade-ideas.com/document/edit?id=1yWmRcGxeYq-4XFzm7cOw-yjnA58bgRpt2brVS1s0xuE&hl=en
 * 
 * This version still does not match the proposed final version:
 *  - Don't map the volume for the ETFs.  Need to check on the special indexes.
 */

class SmbRadarLists : public DataNode
{
private:
  static const int8_t MAX_HIGH_TIME = 5;
  struct CurrentData
  {
  public:
    std::string symbol;
    double priceSpike;
    double volRate;
    double todaysRange;
    double openingRange;
    double strength;
    double upTrend;
    double nearHigh;
    double nearLow;
    int8_t newHigh;
    int8_t newLow;
    typedef std::vector< CurrentData > List;
  };
  class DataType
  { // This represents a window or a pair of windows.  Sometimes two windows
    // are identical except for their sort order.
  public:
    enum Type { PriceSpike, VolRate, TodaysRange, OpeningRange, Strength, UpTrend, NearHigh, NearLow, VolRateMoreStocks };
  private:
    Type _type;  // This can't be const because we want to live in a vector.
    double getSortValue(CurrentData const &data) const;
    static void add(std::string &output, std::string const &fieldName, 
		    std::string const &id, std::string fieldValue);
    static void add(std::string &output, std::string const &fieldName, 
		    std::string const &id, double fieldValue);
    static void add(std::string &output, std::string const &fieldName, 
		    std::string const &id, time_t fieldValue);
    static void add(std::string &output, std::string const &fieldName,
                    std::string const &id, int8_t fieldValue);
  public:
    DataType(Type type = PriceSpike) : _type(type) { }
    bool operator ()(CurrentData const &a, CurrentData const &b) const;
    bool valid(CurrentData const &data) const;
    std::string smallEndName() const;
    std::string bigEndName() const;
    bool includeETFs() const;
    int maxItemCount() const;
    void dump(CurrentData const &, std::string &output, int id) const;
    void dumpAll(CurrentData const &, std::string &output, int id) const;
    static void addSeperator(std::string &output, int id);
    typedef std::vector< DataType > List;
    static void getAll(List &, bool moreStocks);
  };
  class DataProvider : public DataNode
  {
  private:
    const std::string _symbol;
    const double _atr;
    const double _averageBarSize;
    GenericTosDataNode *_tosData;
    GenericDataNode *_relVolData;
    StandardCandles *_candleData;
    BarCounter *_barCounter;
    SmbLRBase *_linearRegressionData;
    double _openingRange;
    double _lastHigh, _lastLow;
    int8_t _newHigh, _newLow;
    enum { wTos, wCandle, wBarCounter };
    void onWakeup(int msgId);
    void onTos();
    void onCandle();
    void onBarCounter();
    double getTodaysRange() const;
    double getPriceSpike() const;
    double getUpTrend() const;

    DataProvider(DataNodeArgument const &args);
    friend class DataNode;

  public:
    static DataNodeLink *find(DataProvider *&node, 
			      std::string const &fileName);
    CurrentData getCurrentData() const;
    typedef std::vector< DataProvider * > List;
  };
  struct MessageCacheItem
  {
    std::string message;
    time_t time;
    MessageCacheItem() : time(0) { }
  };
  typedef std::map< std::string, MessageCacheItem > MessageCache;
  MessageCache _messageCache;
  void possiblySendMessage(std::string const &window, 
			   std::string const &message);
  template < class IT >
  void addFromIterator(std::string &output, int &counter, DataType dataType,
		       IT begin, IT end,
		       int max = std::numeric_limits< int >::max());
  void buildAndSend(DataType dataType,
		    CurrentData::List const &sortedEtfs,
		    CurrentData::List const &sortedOther,
		    bool lowestFirst);
  DataType::List _dataTypes;
  DataProvider::List _etfs, _normalSymbols;
  void checkForMessages();
  void loadList(DataProvider::List &providers, std::string const &listName);

  void onBroadcast(BroadcastMessage &message, int msgId);
  TimerThread::Helper _timer;

  SmbRadarLists(DataNodeArgument const &args);
  friend class DataNode;

public:
  static DataNodeLink *find(bool moreStocks)
  {
    SmbRadarLists *node;
    return findHelper(NULL, 0, node, moreStocks);
  }

};


/////////////////////////////////////////////////////////////////////
// SmbRadarLists::DataType
//
// This is what's interesting about a column or a pair of columns.
// A lot of the formulas are reused over and over, so we store them
// all in a CurrentData object.  The DataType object is only 
// responsible for extracting the data from the CurrentData object.
/////////////////////////////////////////////////////////////////////

double SmbRadarLists::DataType::getSortValue(CurrentData const &data) const
{
  switch (_type)
    {
    case PriceSpike:
      return data.priceSpike;
    case VolRate:
    case VolRateMoreStocks:
      return data.volRate;
    case TodaysRange:
      return data.todaysRange;
    case OpeningRange:
      return data.openingRange;
    case Strength:
      return data.strength;
    case UpTrend:
      return data.upTrend;
    case NearHigh:
      return data.nearHigh;
    case NearLow:
      return data.nearLow;
    }
  // Avoid a strange compiler warning.
  throw false;
}

bool SmbRadarLists::DataType::operator ()(CurrentData const &a, 
					  CurrentData const &b) const
{ // This is used for sorting.  This says if a < b.
  // Our comparisons always use the symbol as the last key field.  That
  // ensures that the sort order is unique.  We don't want two symbols
  // to randomly switch places over time.
  const double aValue = getSortValue(a);
  const double bValue = getSortValue(b);
  if (aValue < bValue)
    return true;
  else if (aValue > bValue)
    return false;
  else
    return a.symbol < b.symbol;

}

bool SmbRadarLists::DataType::valid(CurrentData const &data) const
{ // We typically use this as a filter before sorting.  Otherwise we'd have a
  // lot of bad values at one end or the other.
  if (_type == VolRateMoreStocks)
    return data.volRate > 3.0;
  else
    return std::isfinite(getSortValue(data));
}

std::string SmbRadarLists::DataType::smallEndName() const
{
  switch (_type)
    {
    case Strength:
      return "WeakToday";
    case UpTrend:
      return "DownTrend";
    case NearHigh:
      return "NearHighs";
    case NearLow:
      return "NearLows";
    default:
      return "";
    }
}

std::string SmbRadarLists::DataType::bigEndName() const
{
  switch (_type)
    {
    case PriceSpike:
      return "PriceSpike";
    case VolRate:
      return "InPlay";
    case TodaysRange:
      return "TodaysRange";
    case OpeningRange:
      return "OpeningRange";
    case Strength:
      return "StrongToday";
    case UpTrend:
      return "UpTrend";
    case VolRateMoreStocks:
      return "InPlayMoreStocks";
    default:
      return "";
    }
}

bool SmbRadarLists::DataType::includeETFs() const
{
  switch (_type)
    {
    case VolRate:
    case NearHigh:
    case NearLow:
      return true;
    case PriceSpike:
    case TodaysRange:
    case OpeningRange:
    case Strength:
    case UpTrend:
    case VolRateMoreStocks:
      return false;
    }
  // Avoid a strange compiler warning.
  throw false;
}

int SmbRadarLists::DataType::maxItemCount() const
{
  if (_type == VolRateMoreStocks)
    return 100;
  else
    return 100;
}

void SmbRadarLists::DataType::getAll(List &dest, bool moreStocks)
{
  dest.clear();
  if (moreStocks)
    dest.push_back(VolRateMoreStocks);
  else
    { // There is no ++ defined for enumerated types!
      for (Type type = PriceSpike; type <= NearLow; type = (Type)(type + 1))
	dest.push_back(DataType(type));
    }
}

void SmbRadarLists::DataType::dump(CurrentData const &data, 
				   std::string &output, int id) const
{
  const std::string indexStr = ntoa(id);
  add(output, "symbol", indexStr, data.symbol);
  add(output, "vol_rate", indexStr, data.volRate);
  switch (_type)
    {
    case Strength:
      add(output, "strength", indexStr, data.strength);
      break;
    case VolRate:
    case VolRateMoreStocks:
      // The vol rate column is standard.
      break;
    case NearHigh:
      add(output, "near_high", indexStr, data.nearHigh);
      add(output, "new_high", indexStr, data.newHigh);
      break;
    case NearLow:
      add(output, "near_low", indexStr, data.nearLow);
      add(output, "new_low", indexStr, data.newLow);
      break;
    case PriceSpike:
      add(output, "price_spike", indexStr, data.priceSpike);
      break;
    case TodaysRange:
      add(output, "todays_range", indexStr, data.todaysRange);
      break;
    case OpeningRange:
      add(output, "opening_range", indexStr, data.openingRange);
      break;
    case UpTrend:
      // The requirements said not to add this because it's only meaningful
      // for the sort order, not to read.  If we did decide to display this,
      // we couldn't use the standard add().  We'd have to display more digits.
      // All the results I saw were between -0.2 and 0.2
      // add(output, "up_trend", indexStr, data.upTrend);
      break;
    }
}

void SmbRadarLists::DataType::addSeperator(std::string &output, int id)
{ // The seperator looks just like a normal stock.  The stock name is "-" and
  // all the other fields are blank.  A client doesn't have to do anything
  // special.  It can naively try to display this record and it will look okay.
  // Or the client's display functions can have special logic for this, while
  // the client code for decoding and storing the records treat this like a
  // normal record.
  const std::string indexStr = ntoa(id);
  add(output, "symbol", indexStr, "-");
}

void SmbRadarLists::DataType::dumpAll(CurrentData const &data, 
				      std::string &output, int id) const
{
  const std::string indexStr = ntoa(id);
  add(output, "symbol", indexStr, data.symbol);
  add(output, "price_spike", indexStr, data.priceSpike);
  add(output, "vol_rate", indexStr, data.volRate);
  add(output, "todays_range", indexStr, data.todaysRange);
  add(output, "opening_range", indexStr, data.openingRange);
  add(output, "strength", indexStr, data.strength);
  add(output, "up_trend", indexStr, data.upTrend);
  add(output, "near_high", indexStr, data.nearHigh);
  add(output, "near_low", indexStr, data.nearLow);
  add(output, "new_high", indexStr, data.newHigh);
  add(output, "new_low", indexStr, data.newLow);
}

void SmbRadarLists::DataType::add(std::string &output, 
				  std::string const &fieldName, 
				  std::string const &id, 
				  std::string fieldValue)
{
  if (fieldValue.empty())
    return;
  if (!output.empty())
    output += '&';
  output += urlEncode(fieldName);
  output += id;
  output += '=';
  output += urlEncode(fieldValue);
}

void SmbRadarLists::DataType::add(std::string &output, 
				  std::string const &fieldName, 
				  std::string const &id, 
				  double fieldValue)
{
  if (!std::isfinite(fieldValue))
    return;
  add(output, fieldName, id, dtoaFixed(fieldValue, 1));
}

void SmbRadarLists::DataType::add(std::string &output, 
				  std::string const &fieldName, 
				  std::string const &id, 
				  time_t fieldValue)
{
  if (!fieldValue)
    return;
  add(output, fieldName, id, formatTime(fieldValue));
}

void SmbRadarLists::DataType::add(std::string &output, std::string const &fieldName,
				  std::string const &id, int8_t fieldValue)
{ // The time since the last high/low moves in discrete steps.  We store it in a
  // small integer, but we send it to the client as a ratio.
  if (!fieldValue)
    return;
  add(output, fieldName, id, fieldValue / (double)MAX_HIGH_TIME);
}


/////////////////////////////////////////////////////////////////////
// SmbRadarLists::DataProvider
//
/////////////////////////////////////////////////////////////////////

//static const double NAN = std::numeric_limits< double >::quiet_NaN();
static const double PLUS_INF = std::numeric_limits< double >::infinity();
static const double MINUS_INF = -PLUS_INF;

SmbRadarLists::DataProvider::DataProvider(DataNodeArgument const &args) :
  _symbol(args.getStringValue()),
  _atr(strtodDefault(FileOwnerDataNode::getStringValue
		     ("F_OvernightData.csv", "ATR Q", _symbol),
		     NAN)),
  _averageBarSize(strtodDefault(FileOwnerDataNode::getStringValue
				("F_OvernightData.csv", 
				 "Average Bar Size 5", _symbol),
				NAN)),
  _openingRange(NAN),
  // This is not a valid value, if we ask.  If we try to set high to a new
  // value, it will always be lower than this.  So the first high of the day
  // will be recorded, but we will not report it as a new high.  Any time the
  // high goes down, that's what we do.  So the initial case and a correction
  // are one and the same.
  _lastHigh(PLUS_INF),
  _lastLow(MINUS_INF),
  _newHigh(0),
  _newLow(0)
{
  addAutoLink(GenericTosDataNode::find(this, wTos, _tosData, _symbol));
  addAutoLink(StandardCandles::find(this, wCandle, _candleData, _symbol, 5));
  addAutoLink(BarCounter::find(this, wBarCounter, _barCounter, 3));
  // I copied the 30 periods (2.5 hours) from the EasyLanguage code.  In the
  // discussions we said 2 hours.  But I assume that was an estimate.
  addAutoLink(SmbLRBase::find(_linearRegressionData, _symbol, 5, 30));

  const DataNodeArgument factory =
    GenericDataNodeFactory::findFactory("RelVol");
  addAutoLink(GenericDataNodeFactory::replaceAndFind(factory, NULL, 0,
						     _relVolData,
						     symbolPlaceholder,
						     _symbol));
}

double SmbRadarLists::DataProvider::getTodaysRange() const
{
  TosData const &last = _tosData->getLast();
  if (_tosData->getValid() && (last.high > 0) && (last.low > 0))
    return (last.high - last.low) / _atr * 100.0;
  else
    return NAN;
}

double SmbRadarLists::DataProvider::getUpTrend() const
{
  double m, rr;
  bool valid;
  _linearRegressionData->getSlopeAndRSquared(valid, m, rr);
  if (!valid)
    return NAN;
  if (rr < 0.5)
    // Note that this follows the live EasyLanguage code, not the comments in
    // the code which use a slightly different value.
    return NAN;
  return m;
}

double SmbRadarLists::DataProvider::getPriceSpike() const
{
  int count = 3;
  double biggest = MINUS_INF;
  if (_candleData->isCurrentCandleValid())
    {
      StandardCandles::SingleCandle const &candle = 
	_candleData->getCurrentCandle();
      biggest = candle.high - candle.low;
      //if (biggest < 0.20)
      //return NAN;
      count--;
    }
  StandardCandles::CandleArray const &history = _candleData->getHistory();
  for (StandardCandles::CandleArray::const_reverse_iterator it =
	 history.rbegin();
       count && (it != history.rend());
       count--, it++)
    {
      StandardCandles::SingleCandle const &candle = *it;
      //if ((count == 3) && (candle.high - candle.low < 0.20))
      //return NAN;
      biggest = std::max(biggest, candle.high - candle.low);
    }
  if (biggest < 0.20)
    return NAN;
  return biggest / _averageBarSize;
}

DataNodeLink *SmbRadarLists::DataProvider::find(DataProvider *&node,
						std::string const &fileName)
{ // I made this class a data node for a couple of reasons.  This gives us
  // access to get submit time.  And I can use addAutoLink in this class and
  // in the user of this class to help with cleanup.  We don't provide a
  // callback because everything is done in a timer.
  return findHelper(NULL, 0, node, fileName);
}

SmbRadarLists::CurrentData SmbRadarLists::DataProvider::getCurrentData() const
{
  CurrentData result;
  result.symbol = _symbol;
  result.priceSpike = getPriceSpike();
  bool valid;
  _relVolData->getDouble(valid, result.volRate);
  if (!valid)
    result.volRate = NAN;
  TosData const &last = _tosData->getLast();
  result.todaysRange = getTodaysRange();
  result.openingRange = _openingRange;
  if (_tosData->getValid() && (last.price > 0) && (last.open > 0))
    result.strength = (last.price - last.open) / _atr * 100.0;
  else
    result.strength = NAN;
  result.upTrend = getUpTrend();
  if (_tosData->getValid() && (last.price > 0) && (last.high > 0))
    result.nearHigh = (last.high - last.price) / _atr * 100.0;
  else
    result.nearHigh = NAN;
  if (_tosData->getValid() && (last.price > 0) && (last.low > 0))
    result.nearLow = (last.price - last.low) / _atr * 100.0;
  else
    result.nearLow = NAN;
  result.newHigh = _newHigh;
  result.newLow = _newLow;
  return result;
}

void SmbRadarLists::DataProvider::onWakeup(int msgId)
{
  switch (msgId)
    {
    case wTos:
      onTos();
      break;
    case wCandle:
      onCandle();
      break;
    case wBarCounter:
      onBarCounter();
      break;
    }
}

void SmbRadarLists::DataProvider::onTos()
{
  if (!_tosData->getValid())
    return;
  TosData const &last = _tosData->getLast();
  if ((last.high <= 0) || (last.low <= 0))
    return;
  if (last.high > _lastHigh)
    _newHigh = MAX_HIGH_TIME;
  _lastHigh = last.high;
  if (last.low < _lastLow)
    _newLow = MAX_HIGH_TIME;
  _lastLow = last.low;
}

void SmbRadarLists::DataProvider::onCandle()
{
  if (secondOfTheDay(_candleData->getCurrentCandleStartTime()) == 
      MARKET_HOURS_OPEN + 15 * MARKET_HOURS_MINUTE)
    _openingRange = getTodaysRange();  
}

void SmbRadarLists::DataProvider::onBarCounter()
{
  if (_barCounter->getTimePhase() != BarCounter::tpNotify)
    return;
  if (_newHigh > 0)
    _newHigh--;
  if (_newLow > 0)
    _newLow--;
}
  

/////////////////////////////////////////////////////////////////////
// SmbRadarLists
/////////////////////////////////////////////////////////////////////

void SmbRadarLists::possiblySendMessage(std::string const &window,
					std::string const &message)
{ // The cache is important.  From what I saw, the screen was barely ever
  // updating.
  const time_t now = time(NULL);
  MessageCacheItem &cached = _messageCache[window];
  if ((now - cached.time >= 30) || (cached.message != message))
    {
      cached.time = now;
      cached.message = message;
      ReportAlertsThread::getInstance()->reportAlert("SMB-" + window, message);
    }
}

template < class IT >
void SmbRadarLists::addFromIterator(std::string &output, 
				    int &counter, DataType dataType, 
				    IT begin, IT end, 
				    int max)
{
  while ((begin != end) && max)
    {
      dataType.dump(*begin, output, counter);
      begin++;
      max--;
      counter++;
    }
}

void SmbRadarLists::buildAndSend(DataType dataType, 
				 CurrentData::List const &sortedEtfs,
				 CurrentData::List const &sortedOther,
				 bool lowestFirst)
{
  std::string window;
  if (lowestFirst)
    window = dataType.smallEndName();
  else
    window = dataType.bigEndName();
  if (window.empty())
    return;
  int counter = 0;
  std::string message;
  if (lowestFirst)
    addFromIterator(message, counter, dataType, 
		    sortedEtfs.begin(), sortedEtfs.end());
  else
    addFromIterator(message, counter, dataType,
                    sortedEtfs.rbegin(), sortedEtfs.rend());
  if (!(sortedEtfs.empty() || sortedOther.empty()))
    { // If both sections have data, we need a seperator between them.  If
      // either or both are empty then there is no need to add a seperator.
      DataType::addSeperator(message, counter);
      counter++;
    }
  if (lowestFirst)
    addFromIterator(message, counter, dataType,
                    sortedOther.begin(), sortedOther.end(), 
		    dataType.maxItemCount());
  else
    addFromIterator(message, counter, dataType,
                    sortedOther.rbegin(), sortedOther.rend(), 
		    dataType.maxItemCount());
  possiblySendMessage(window, message);
}

void SmbRadarLists::checkForMessages()
{
  if (secondOfTheDay(time(NULL)) > MARKET_HOURS_CLOSE + 5)
    // Freeze the output at the close.  More precisely 5 seconds later to make
    // sure we don't stop a few seconds before.
    return;

 // There's a lot of overlap in the formulas.  So we compute all of the fields
  // all at once.
  CurrentData::List etfs, otherStocks;
  for (DataProvider::List::const_iterator it = _etfs.begin();
       it != _etfs.end();
       it++)
    etfs.push_back((*it)->getCurrentData());
  for (DataProvider::List::const_iterator it =  _normalSymbols.begin();
       it != _normalSymbols.end();
       it++)
    otherStocks.push_back((*it)->getCurrentData());
  // Then we look at each data type and filter and sort the items according to
  // each window type.
  for (DataType::List::const_iterator dataTypeIt = _dataTypes.begin();
       dataTypeIt != _dataTypes.end();
       dataTypeIt++)
    {
      DataType const &dataType = *dataTypeIt;
      // We create new lists, so we can start from the same unfiltered data
      // for each window.
      CurrentData::List validEtfs, validOtherStocks;
      if (dataType.includeETFs())
	{
	  for (CurrentData::List::const_iterator it = etfs.begin();
	       it != etfs.end();
	       it++)
	    if (dataType.valid(*it))
	      validEtfs.push_back(*it);
	  std::sort(validEtfs.begin(), validEtfs.end(), dataType);
	}
      for (CurrentData::List::const_iterator it = otherStocks.begin();
	   it != otherStocks.end();
	   it++)
	if (dataType.valid(*it))
	  validOtherStocks.push_back(*it);
      std::sort(validOtherStocks.begin(), validOtherStocks.end(), dataType);
      // Sending the top of the list is very similar to sending the bottom of
      // the list.  We don't resort if that's the only difference between the
      // two lists.
      buildAndSend(dataType, validEtfs, validOtherStocks, true);
      buildAndSend(dataType, validEtfs, validOtherStocks, false);
    }
}

void SmbRadarLists::loadList(DataProvider::List &providers, 
			     std::string const &listName)
{
  std::ifstream stream(listName.c_str(), std::ios_base::in);
  while(stream)
    {
      std::string symbol;
      std::getline(stream, symbol);
      symbol = strtoupper(trim(symbol, " \r"));
      if (!symbol.empty())
	{ 
	  DataProvider *node;
	  addAutoLink(DataProvider::find(node, symbol));
	  providers.push_back(node);
	}
    }
}

SmbRadarLists::SmbRadarLists(DataNodeArgument const &args) :
  _timer(getOwnerChannel(), &getManager()->getTimerThread())
{
  const bool moreStocks = args.getBooleanValue();
  DataType::getAll(_dataTypes, moreStocks);
  for (DataType::List::const_iterator it = _dataTypes.begin();
       it != _dataTypes.end();
       it++)
    {
      std::string name = it->smallEndName();
      if (!name.empty())
	ReportAlertsThread::getInstance()->setCacheable("SMB-" + name);
      name = it->bigEndName();
      if (!name.empty())
	ReportAlertsThread::getInstance()->setCacheable("SMB-" + name);
    }
  loadList(_etfs, "data/smb_etf_symbols.txt");
  if (moreStocks)
    loadList(_normalSymbols, "data/smb_more_symbols.txt");
  else
    {
      loadList(_normalSymbols, "data/smb_normal_symbols.txt");
    }
  registerForBroadcast(_timer, 0);
  getManager()->getTimerThread().requestPeriodicBroadcast(_timer, 5000);
}

void SmbRadarLists::onBroadcast(BroadcastMessage &message, int msgId)
{ // Timer!
  checkForMessages();
}


/////////////////////////////////////////////////////////////////////
// Global
/////////////////////////////////////////////////////////////////////

static std::vector< DataNodeLink * > cleanup;

void initSmbRadar()
{
  cleanup.push_back(SmbRadarLists::find(true));
  cleanup.push_back(SmbRadarLists::find(false));
}