/* This was the 3rd version of the running alert.  That's why we called it
 * "intermediate".  It has the advantage of the confirmed running because it
 * can look at things on different timescales.  It has the advantage of the
 * short term running because it can be triggered much faster.  It looks at
 * the bid and ask, like the short term running, to get some confirmation.
 *
 * The basic idea is that we are looking at 15 minutes worth of 30 second bars
 * for a historical baseline.  We are comparing the current NBBO to any
 * extremes we saw in these bars.
 *
 * We do not keep all 30 bars that would be required to do this exactly.  
 * Instead we compressed the data and remove any bars that are not interesting.
 * Of course, we delete any bars that are too old to meet our criteria.  Also,
 * someimtes the stock price moves in in uninteresting direction.  We can tell
 * from context that one or more bars would never be the most interesting, so
 * we delete those.  If we still need more room -- we hard code the maximum
 * length to 7 bars -- then we combine two bars.  We use an exponential system
 * to decide which bars to keep.  The idea is to have more detail for the more
 * recent history, and less detail for the older bars.
 */

#include <math.h>
#include <limits>

#include "../data_framework/GenericTosData.h"
#include "../data_framework/GenericL1Data.h"
#include "../../shared/MarketHours.h"
#include "../data_framework/SimpleMarketData.h"
#include "../alert_framework/AlertBase.h"
#include "../../shared/LogFile.h"

#include "IntermediateRunning.h"

class IntermediateRunning : public Alert
{
private:
  struct HistoricalCandle
  {
    double extreme;
    double maxPreviousStrength;
    time_t time;
  };
  struct RecentHistory
  {
    static const int MAX = 7;
    HistoricalCandle candle[MAX];
    int_fast8_t count;
    RecentHistory() : count(0) { }
  };
  enum Comparison { cLessReportable, cEqual, cMoreReportable };
  struct CurrentGoal
  { // This is the last point we are comparing to when we determine the
    // quality.
    double extreme;
    // We display this point, which may be slighly higher.  We discount the
    // spread when calculating the quality, but not when displaying the result.
    double toDisplay;
    // The first time that the Extreme gets higher, we record that price, the
    // that price plus the spread, and the time.
    time_t time;
    CurrentGoal() : extreme(0.0), toDisplay(0.0), time(0) { }
  };
  struct CurrentSource
  { // This is the point that we will record for this candle as a possible
    // starting place.
    double extreme;
    // The is the last time that we saw this piece.
    time_t time;
    CurrentSource() : extreme(0.0), time(0) { }
  };

  bool _up;
  double _volatility;  // This is 0.0 if we don't have any data.
  GenericL1DataNode *_l1Data;

  RecentHistory _recentHistory;
  CurrentGoal _currentGoal;
  CurrentSource _currentSource;
  bool _lastSourceWasExtreme;
  bool _currentValid;
  time_t _lastPushTime;
  
  void clearData();
  enum { wL1, wTOS };
  void onWakeup(int msgId);
  void compressRecentHistory();
  void checkForAlert();
  void updateCandles(bool timeOnly);
  Comparison compare(double older, double newer);
  double difference(double older, double newer);
  void removeItem(int index);

  static const int CANDLE_WIDTH = 30;  // 30 seconds.

  IntermediateRunning(DataNodeArgument args);
  friend class GenericDataNodeFactory;
};

void IntermediateRunning::clearData()
{
  _recentHistory.count = 0;
  _currentValid = false;
}

void IntermediateRunning::onWakeup(int msgId)
{
  switch (msgId)
    {
    case wL1:
      updateCandles(false);
      break;
    case wTOS:
      updateCandles(true);
      break;
    }
}

void IntermediateRunning::removeItem(int index)
{
  assert((index > 0) && (index < RecentHistory::MAX));
  /* We could not pass this point (going left) unless the strength was
   * higher than this.  The implication is that the limit for everything
   * further left is at least this high.  When we delete this entry,
   * we still don't want anyone to get past this point without the
   * required strength. */
  _recentHistory.candle[index - 1].maxPreviousStrength =
    std::max(_recentHistory.candle[index - 1].maxPreviousStrength,
	     _recentHistory.candle[index].maxPreviousStrength);
  for (int i = index; i < RecentHistory::MAX - 1; i++)
    _recentHistory.candle[i] = _recentHistory.candle[i + 1];
  _recentHistory.count--;
}

void IntermediateRunning::compressRecentHistory()
{
  if ((_recentHistory.count != RecentHistory::MAX) || !_currentValid)
    // We're not full.  This could be an assertion failed.  But if we're not
    // full, we won't do the push.  Still, we don't expect it. 
    return;
  int weight = 1;
  time_t bestScore = std::numeric_limits< time_t >::max();
  int bestIndex = -1;
  // Look at each gap.
  for (int i = 1; i < RecentHistory::MAX; i++)
    {
      const time_t currentScore = 
	(_recentHistory.candle[i].time - _recentHistory.candle[i-1].time) 
	* weight;
      weight <<= 1;
      if (currentScore < bestScore)
	{
	  bestScore = currentScore;
	  bestIndex = i;
	}
    }
  if (bestIndex == 1)
    // If we want to remove the first, oldest gap, we always remove the
    // later print.  This keeps the age of the oldest print from shrinking.
    // We couldn't use the algorithm below because we'd be reading off the
    // left side of the array. 
    removeItem(1);
  else
    { // To remove the gap, we have to remove one of the two items
      // immediately adjacent to the gap.  We have to choose which of these
      // two items to keep.  We look at the next item on either side of these
      // items.  We try keep the item which is most centered relative to the
      // items around it. 
      double comparisonPrice;
      if (bestIndex == RecentHistory::MAX - 1)
	// To deal with the most current historical item, we have to compare
	// it to the item which is building, and not yet in the history yet.
	comparisonPrice = _currentSource.extreme;
      else
	comparisonPrice = _recentHistory.candle[bestIndex + 1].extreme;
      comparisonPrice = (_recentHistory.candle[bestIndex - 2].extreme 
			 + comparisonPrice) / 2;
      if (fabs(comparisonPrice - _recentHistory.candle[bestIndex].extreme)
	  >= fabs(comparisonPrice - _recentHistory.candle[bestIndex - 1].extreme))
	//The later one is further from the average, so delete it.
	removeItem(bestIndex);
      else
	// The earlier one gets it.
	removeItem(bestIndex - 1);
    }
}

/* Check on time.  See if we need to move data from the current candle to the
 * historical candles, etc.  If we make the move, then we check for an alert,
 * and then we create a new current candle.  Otherwise, assuming that TimeOnly
 * is false, we update the current candle from the L1.  TimeOnly is an
 * optimization so we don't have to do as much work if we are called from a
 * timer, or a TOS update. */
void IntermediateRunning::updateCandles(bool timeOnly)
{
  static const int MAX_HISTORY = 25 * MARKET_HOURS_MINUTE;
  if (_volatility <= 0)
    return;
  const time_t currentTime = getSubmitTime();
  bool dataValid;
  if (timeOnly)
    dataValid = _currentValid;
  else
    {
      if (_l1Data->getValid())
	dataValid = (_l1Data->getCurrent().bidPrice > 0) 
	  && (_l1Data->getCurrent().askPrice > 0);
      else
	dataValid = false;
      if (!dataValid)
	clearData();
    }
  if (dataValid)
    {
      bool startNewCandle = !_currentValid;
      if (_currentValid && (currentTime - _lastPushTime >= CANDLE_WIDTH))
	{ // Push the candle into the history list and start a new candle. 
	  // Start by deleting recent candles which will be obscured
	  // by the new candle. 
	  int i = _recentHistory.count - 1;
	  while ((i >= 0) 
		 && (compare(_recentHistory.candle[i].extreme,
			     _currentSource.extreme) != cMoreReportable))
	    {
	      i--;
	      if (i >= 0)
		_recentHistory.candle[i].maxPreviousStrength =
		  std::max(_recentHistory.candle[i].maxPreviousStrength,
			   _recentHistory.candle[i + 1].maxPreviousStrength);
	    }
	  _recentHistory.count = i + 1;
	  // Delete anything that is too old.  Should be one at most one
	  // item at a time, so let's keep it simple.
	  if ((_recentHistory.count > 0) 
	      && (_recentHistory.candle[0].time < currentTime - MAX_HISTORY))
	    {
	      _recentHistory.count--;
	      for (int j = 1; j <= _recentHistory.count; j++)
		_recentHistory.candle[j - 1] = _recentHistory.candle[j];
	    }
	  // If we still don't have room in the list, we need to combine
	  // two entries to make room. 
	  if (_recentHistory.count == RecentHistory::MAX)
	    compressRecentHistory();
	  // If the extreme price was also the closing price, update the
	  // time accordingly. 
	  if (_lastSourceWasExtreme)
	    _currentSource.time = currentTime;	 
	  { // Store the current candle at the end of the list.
	    HistoricalCandle &candle = 
	      _recentHistory.candle[_recentHistory.count];
	    candle.extreme = _currentSource.extreme;
	    candle.time = _currentSource.time;
	    candle.maxPreviousStrength = 0;
	  }
	  _recentHistory.count++;
	  // Now compare the current data to the new history.
	  checkForAlert();
	  // Start a new candle.
	  startNewCandle = true;
	}
      if (startNewCandle || !timeOnly)
	{
	  L1Data const &current = _l1Data->getCurrent();
	  double goalValue;
	  double sourceValue;
	  if (compare(current.bidPrice, current.askPrice) == cMoreReportable)
	    { // The spread always works against us.  We choose these values so
	      // that the direction from the SourceValue to the GoalValue is
	      // the opposite of what the alert is looking for. 
	      goalValue = current.bidPrice;
	      sourceValue = current.askPrice;
	    }
	  else
	    {
	      goalValue = current.askPrice;
	      sourceValue = current.bidPrice;
	    }
	  if (startNewCandle)
	    { // Create a new pair of current candles.
	      _lastPushTime = currentTime;
	      _currentGoal.extreme = goalValue;
	      _currentGoal.toDisplay = sourceValue;
	      _currentGoal.time = currentTime;
	      _currentSource.extreme = sourceValue;
	      _currentSource.time = currentTime;
	      _lastSourceWasExtreme = true;
	      _currentValid = true;
	    }
	  else if (!timeOnly)
	    { // Update the current candles. 
	      if (compare(_currentGoal.extreme, goalValue) == cMoreReportable)
		{
		  _currentGoal.extreme = goalValue;
		  _currentGoal.toDisplay = sourceValue;
		  _currentGoal.time = currentTime;
		}
	      if (compare(_currentSource.extreme, sourceValue) 
		  != cMoreReportable)
		{
		  _currentSource.extreme = sourceValue;
		  _currentGoal.time = currentTime;
		  _lastSourceWasExtreme = true;
		}
	      else
		_lastSourceWasExtreme = false;
	    }
	  
	}
    }
}

void IntermediateRunning::checkForAlert()
{
  static const double MIN_REPORTABLE_STRENGTH = 2.0;
  if (!_currentValid)
    return;
  if (getSubmitTime() - _currentGoal.time > 2 * CANDLE_WIDTH)
    /* Event is old.  Displaying an alert would be confusing at best.  This
     * can only happen when a stock is barely active, so it's okay to throw
     * the alert away.  Note:  This value can easily be one candle width even
     * if we have a lot of prints, if the goal value went up right at the
     * beginning of the candle.  That's why we allow 2 candle widths.  */
    return;
  // Find best.
  bool success = false;
  static const int NO_VALUE = -1;
  int bestIndex = NO_VALUE;
  time_t startTime = 0;
  time_t endTime = _currentGoal.time;
  double priceChange = 0.0;
  double strength = 0.0;
  { 
    /* The volatility is expressed in the number of $ the price can be
     * expected to move in 15 minutes.  We are looking at smaller times,
     * so we scale things to the right time.  This was we know what values
     * we should consider unusual. */
    static const int TIME_UNIT = MARKET_HOURS_MINUTE * 15;
    int lastComparison = NO_VALUE;
    for (int currentIndex = 0; 
	 currentIndex < _recentHistory.count; 
	 currentIndex++)
      {
	HistoricalCandle const &candle = _recentHistory.candle[currentIndex];
	if (candle.time + CANDLE_WIDTH < _currentGoal.time)
	  {
	    double currentStrength = 
	      difference(candle.extreme, _currentGoal.extreme) 
	      / sqrt((endTime - candle.time) / (double)TIME_UNIT)
	      / _volatility;
	    if (!finite(currentStrength))
	      { // This should be moot.  I think this bug is fixed.
		TclList debug;
		debug<<"IntermediateRunning.C"
		     <<"IntermediateRunning::checkForAlert()"
		     <<"currentStrength"
		     <<currentStrength
		     <<"candle.extreme"
		     <<candle.extreme
		     <<"_currentGoal.extreme"
		     <<_currentGoal.extreme
		     <<"endTime"
		     <<endTime
		     <<"candle.time"
		     <<candle.time
		     <<"_volatility"
		     <<_volatility;
		LogFile::primary().sendString(debug);
	      }
	    if ((bestIndex == NO_VALUE) || (currentStrength > strength))
	      {
		strength = currentStrength;
		bestIndex = currentIndex;
	      }
	    if (bestIndex != NO_VALUE)
	      if (strength <= candle.maxPreviousStrength)
		bestIndex = NO_VALUE;
	    lastComparison = currentIndex;
	  }
      }
    if (bestIndex == NO_VALUE)
      success = false;
    else
      {
	success = true;
	startTime = _recentHistory.candle[bestIndex].time;
	priceChange =
	  _currentGoal.toDisplay - _recentHistory.candle[bestIndex].extreme;
	_recentHistory.candle[lastComparison].maxPreviousStrength = 
	  std::max(_recentHistory.candle[lastComparison].maxPreviousStrength,
		   strength);
      }
  }
  if (success && (strength > MIN_REPORTABLE_STRENGTH))
    {
      std::string msg = "Running ";
      if (_up)
	msg += "up";
      else
	msg += "down";
      msg += ": ";
      msg += formatPrice(priceChange, true);
      msg += " in ";
      msg += durationString(startTime, endTime);
      std::string altMsg = "p=";
      altMsg += formatPrice(priceChange);
      altMsg += "&d=";
      altMsg += ntoa(endTime - startTime);
      report(msg, altMsg, strength);
    }
}

IntermediateRunning::Comparison 
IntermediateRunning::compare(double older, double newer)
{
  if (newer == older)
    return cEqual;
  if ((newer > older) == _up)
    return cMoreReportable;
  return cLessReportable;
}

double IntermediateRunning::difference(double older, double newer)
{
  if (_up)
    return newer - older;
  return older - newer;
}

IntermediateRunning::IntermediateRunning(DataNodeArgument args) :
  _l1Data(NULL),
  _lastSourceWasExtreme(false),
  _currentValid(false),
  _lastPushTime(0)
{
  DataNodeArgumentVector const &argList = args.getListValue();
  assert(argList.size() == 2);  // (Symbol, up)
  std::string const &symbol = argList[0].getStringValue();
  _up = argList[1].getBooleanValue();
  _volatility = getTickVolatility(symbol);
  if (_volatility < MIN_VOLATILITY)
    return;
  addAutoLink(GenericL1DataNode::find(this, wL1, _l1Data, symbol));
  // We don't use the TOS data.  We just use this as a timer.
  GenericTosDataNode *unused;  
  addAutoLink(GenericTosDataNode::find(this, wTOS, unused, symbol));
}

void initializeIntermediateRunning()
{
  GenericDataNodeFactory::sf< IntermediateRunning >
    ("RunningUpIntermediate", symbolPlaceholderObject, true);
  GenericDataNodeFactory::sf< IntermediateRunning >
    ("RunningDownIntermediate", symbolPlaceholderObject, false);
}