#include #include "../misc_framework/GenericDataNodes.h" #include "GenericTosData.h" #include "../../shared/MarketHours.h" #include "../../shared/SimpleLogFile.h" #include "ShortTermCandles.h" /* This unit is aimed at handling requests like "show me stocks that have moved up $0.50 in the last 5 minutes" and "show me stocks which have had unusual volume in the last 10 minutes". These are grouped togather because they use the same underlying picture of the historical data: TShortTermCandles. This view is simpler and more traditional than a lot of what we've done in the past. Each of these is precise to the minute. If we are exactly at a minute boundary then everything is precise. Otherwise we could be up to one minute off. The running up now alert works a similar way, but doesn't use any history. */ //////////////////////////////////////////////////////////////////// // ShortTermCandles // // TShortTermCandles provides a slightly different view of a candlestick // chart than we use in other places. This only has today's data. It // starts out empty. It always keeps 1 minute candles. It keeps a // fixed (maximum) number of candles. It does not distinguish between // pre, post, and normal market hours. // // If there are periods of no activity we set the volume of the candle // to 0 and the high, low, open and close of the candle to the closing // value of the previous candle. // // I recenently changed this so it could look at other time frames. // Typically this stored one minute candles, so everything else was // precise to the minute. Now you can specify the candle size in // seconds. So you can have more or less precision. // // The expected use is for everything that looks at 5 minutes or more // to be precise to the minute, and everything that looks at 2 minutes // or less to be precise to the second. //////////////////////////////////////////////////////////////////// // Originally we set this because some consumers were looking // for two hours worth of 1 minute candles. Later we added 1 minute // items. The original plan was to make them precise to 10 seconds // or so, but it was hard to change this constant, so we were already // doing the work, so we decided to make this precise to the second. // By the same logic, we already had people looking at the 2 minute // volume (Shane Bird), and since we were already storing 120 candles, // we converted that from 2 one-minute candles to 120 one-second // candles. static const int MAX_HISTORY = 120; // We keep 120 candles, no more. struct SingleCandle { double lastPrice, highPrice, lowPrice; DataNode::Integer endingVolume; }; class ShortTermCandles : public DataNode { private: GenericTosDataNode *_tosData; int_fast8_t _secondsPerBar; SingleCandle _history[MAX_HISTORY]; int_fast16_t _historyCount; int_fast16_t _historyNextIndex; // If we add a candle, it goes here. When we want the last candle, it is one before here, possibly wrapping around. bool _currentInitialized; SingleCandle _current; time_t _currentEndTime; time_t _latestPrintTime; int _epoch; void addCandle(SingleCandle const &newCandle); void extendCandles(int count); void onWakeup(int msgId); ShortTermCandles(DataNodeArgument const &args); friend class DataNode; public: static DataNodeLink *find(DataNodeListener *listener, int msgId, ShortTermCandles *&node, std::string const &symbol, int secondsPerBar = MARKET_HOURS_MINUTE); void checkHistoryState(); SingleCandle const &getCandle(int back) const; SingleCandle const &getCandleOrFirst(int back) const; int_fast16_t getHistoryCount() const { return _historyCount; } SingleCandle const &getCurrentCandle() const { return _current; } bool currentCandleValid() const { return _currentInitialized; } bool tosValid() const { return _tosData->getValid(); } TosData const &tosValue() const { return _tosData->getLast(); } std::string const &symbol() const { return _tosData->symbol(); } int getEpoch() const { return _epoch; } }; ShortTermCandles::ShortTermCandles(DataNodeArgument const &args) : _historyCount(0), _historyNextIndex(0), _currentInitialized(false), _currentEndTime(0), _latestPrintTime(0), _epoch(0) { DataNodeArgumentVector const &argList = args.getListValue(); assert(argList.size() == 2); // Expected params: (Symbol, seconds / bar) _secondsPerBar = argList[1].getIntValue(); addAutoLink(GenericTosDataNode::find(this, 0, _tosData, argList[0].getStringValue())); } DataNodeLink *ShortTermCandles::find(DataNodeListener *listener, int msgId, ShortTermCandles *&node, std::string const &symbol, int secondsPerBar) { return findHelper(listener, msgId, node, argList(symbol, secondsPerBar)); } // We keep a fixed maximum number of candles. We add one candle at a time to // the end. We delete candles from the beginning, if required, to maintian // the maximum length. To avoid unnecessary copies, we always write the new // value over the old value that is disappearing, and we update a pointer to // the end of the list. void ShortTermCandles::addCandle(SingleCandle const &newCandle) { _history[_historyNextIndex] = newCandle; if (_historyCount < MAX_HISTORY) _historyCount++; _historyNextIndex = (_historyNextIndex+1) % MAX_HISTORY; } // There was a period of inactivity. Fill in the blanks. 3 candles back // should always be 3 minutes ago, even if nothing happened since then. void ShortTermCandles::extendCandles(int count) { if (_historyCount > 0) { if (count > MAX_HISTORY) count = MAX_HISTORY; SingleCandle toCopy = getCandle(1); toCopy.highPrice = toCopy.lastPrice; toCopy.lowPrice = toCopy.lastPrice; for (int i = 1; i <= count; i++) addCandle(toCopy); } } SingleCandle const &ShortTermCandles::getCandle(int back) const { assert((back <= _historyCount) && (back > 0)); return _history[(_historyNextIndex - back + MAX_HISTORY) % MAX_HISTORY]; } SingleCandle const &ShortTermCandles::getCandleOrFirst(int back) const { return getCandle(std::min((int)_historyCount, back)); } // The data can update for two reasons. // 1) If there is a new print then we might have to update the current candle. // 2) As time passes we update the older candles. // a) We might demote the current candle to the list of historical candles. // b) We might get rid of very old historical candles. // This procedure takes care of #2, the time part of the algorithm. void ShortTermCandles::checkHistoryState() { time_t currentTime = getSubmitTime(); if (currentTime >= _currentEndTime) { _epoch++; // We don't really do a good job for just any value of _secondsPerBar. // Really we're only expecting one 1 or one minute. time_t newEndTime; if (_secondsPerBar < 60) { // Do not round off. newEndTime = currentTime + _secondsPerBar; } else { // Go to the next minute. Add anywhere between one second and one minute // to do that. For simplicity ignore leap seconds. struct tm brokenDown; localtime_r(¤tTime, &brokenDown); newEndTime = currentTime - brokenDown.tm_sec + _secondsPerBar; } int barsPassed = 0; if (_currentEndTime > 0) barsPassed = (newEndTime - _currentEndTime) / _secondsPerBar; if (_currentInitialized) { // What was the current candle is now a historical candle. addCandle(_current); _currentInitialized = false; barsPassed = std::max(0, barsPassed-1); } extendCandles(barsPassed); _currentEndTime = newEndTime; } } void ShortTermCandles::onWakeup(int msgId) { if (!_tosData->getValid()) return; // If the time goes backwards, ignore the current print. Assume it // was reported late. time_t currentTime = getSubmitTime(); if (currentTime < _latestPrintTime) return; _latestPrintTime = currentTime; checkHistoryState(); TosData const &last = _tosData->getLast(); if (!last.price) // We get a surprising number of 0's here. I often see the range for the // stock being the same as the price in the early morning. This is most // visible on the stocks like this web page, but it can cause problems in // other places. return; _current.lastPrice = last.price; _current.endingVolume = last.volume; if (_currentInitialized) { _current.highPrice = std::max(_current.highPrice, _current.lastPrice); _current.lowPrice = std::min(_current.lowPrice, _current.lastPrice); } else { _current.highPrice = _current.lastPrice; _current.lowPrice = _current.lastPrice; } _currentInitialized = true; // We only notify listeners when a valid print appears. Ideally we // should also notify them when the time updates. The "standard // candles" do that. We ignore that case for simplicity. We can // update at other times, on request, but I don't think there's any // need for us to signal an event at any other time. I feel safe // saying that because I know which other classes are using this. notifyListeners(); } //////////////////////////////////////////////////////////////////// // TimedMovement // // This data node tells you how much the stock has moved in the last N // minutes. // // "the last N minutes" really means the last N candles, including // the candle in progress. The change in the last N candles will be // the change in the last N-1 to N minutes. // // This is based on ShortTermCandles and has the same limitations. // It will notify you on each print. It will give you the right value // at other times, but it will not notify you of clock events. It is // only precise to the minute. // // Recently we changed this to be precise to the second if we are // looking at 1 or 2 minutes. //////////////////////////////////////////////////////////////////// class TimedMovement : public GenericDataNode { private: ShortTermCandles *_candles; int_fast16_t _candleCount; TimedMovement(DataNodeArgument const &args); friend class GenericDataNodeFactory; public: void getDouble(bool &valid, double &value) const; }; TimedMovement::TimedMovement(DataNodeArgument const &args) { DataNodeArgumentVector const &argList = args.getListValue(); assert(argList.size() == 2); // Expected params: (Symbol, Count) const int minutes = argList[1].getIntValue(); int secondsPerCandle; if (minutes <= 2) { // Precise to the second _candleCount = minutes * MARKET_HOURS_MINUTE; secondsPerCandle = 1; } else { // Precise to the minute _candleCount = minutes; secondsPerCandle = MARKET_HOURS_MINUTE; } addAutoLink(ShortTermCandles::find(this, 0, _candles, argList[0].getStringValue(), secondsPerCandle)); } void TimedMovement::getDouble(bool &valid, double &value) const { _candles->checkHistoryState(); valid = _candles->getHistoryCount() && _candles->tosValid(); if (valid) value = _candles->tosValue().price - _candles->getCandleOrFirst(_candleCount).lastPrice; } //////////////////////////////////////////////////////////////////// // VolumeChange // // This data node tells you how much volume the stock has traded in the // last N minutes. // // "the last N minutes" really means the last N candles, including // the candle in progress. The change in the last N candles will be // the change in the last N-1 to N minutes. // // This is based on ShortTermCandles and has the same limitations. // It will notify you on each print. It will give you the right value // at other times, but it will not notify you of clock events. It is // only precise to the minute. //////////////////////////////////////////////////////////////////// class VolumeChange : public GenericDataNode { private: ShortTermCandles *_candles; int_fast16_t _candleCount; VolumeChange(DataNodeArgument const &args); friend class GenericDataNodeFactory; public: void getInteger(bool &valid, Integer &value) const; }; VolumeChange::VolumeChange(DataNodeArgument const &args) { DataNodeArgumentVector const &argList = args.getListValue(); assert(argList.size() == 2); // Expected params: (Symbol, Count) const int minutes = argList[1].getIntValue(); int secondsPerCandle; if (minutes <= 2) { // Precise to the second _candleCount = minutes * MARKET_HOURS_MINUTE; secondsPerCandle = 1; } else { // Precise to the minute _candleCount = minutes; secondsPerCandle = MARKET_HOURS_MINUTE; } addAutoLink(ShortTermCandles::find(this, 0, _candles, argList[0].getStringValue(), secondsPerCandle)); } void VolumeChange::getInteger(bool &valid, Integer &value) const { _candles->checkHistoryState(); valid = _candles->getHistoryCount() && _candles->tosValid(); if (!valid) return; value = _candles->tosValue().volume - _candles->getCandleOrFirst(_candleCount).endingVolume; // Our algorithm is less than perfect because sometimes, when we first // start, we are looking at yesterday's volume and we don't know it. So // at some time the volume will drop very sharply and give us a negative // number. At that point the right value for this filter can only be "I // don't know". valid = value >= 0; } //////////////////////////////////////////////////////////////////// // TimedMovementPercent // // This data node tells you how much the stock has moved in the last N // minutes. // // This is based on TimedMovement and has the same limitations. // It will notify you on each print. It will give you the right value // at other times, but it will not notify you of clock events. It is // only precise to the minute. // // TimedMovement sends the the actual movement, the difference between // the old value and the current value, to the database. We are sending // the current value to the database in a different piece of code. When // the user asks for the percent change, the database converts the value // to a percent. In this case the user only has the percent option. // This is aimed at certain indexes like the S&P 500. We use percent, // in part, so that the users don't know exactly which indicator we are // looking at. An ETF and various forms of the futures will all tell a // similar story to the actual index, but they all have a different // scale. By only showing the user a percentage, the scale should not // matter. //////////////////////////////////////////////////////////////////// class TimedMovementPercent : public GenericDataNode { private: ShortTermCandles *_candles; int_fast16_t _candleCount; TimedMovementPercent(DataNodeArgument const &args); friend class GenericDataNodeFactory; public: void getDouble(bool &valid, double &value) const; }; TimedMovementPercent::TimedMovementPercent(DataNodeArgument const &args) { DataNodeArgumentVector const &argList = args.getListValue(); assert(argList.size() == 2); // Expected params: (Symbol, Count) const int minutes = argList[1].getIntValue(); int secondsPerCandle; if (minutes <= 2) { // Precise to the second _candleCount = minutes * MARKET_HOURS_MINUTE; secondsPerCandle = 1; } else { // Precise to the minute _candleCount = minutes; secondsPerCandle = MARKET_HOURS_MINUTE; } addAutoLink(ShortTermCandles::find(this, 0, _candles, argList[0].getStringValue(), secondsPerCandle)); } void TimedMovementPercent::getDouble(bool &valid, double &value) const { _candles->checkHistoryState(); valid = _candles->getHistoryCount() && _candles->tosValid(); if (!valid) return; double originalPrice = _candles->getCandleOrFirst(_candleCount).lastPrice; if (originalPrice == 0.0) valid = false; else value = _candles->tosValue().price * 100.0 / originalPrice - 100.0; } //////////////////////////////////////////////////////////////////// // IntraDayRange // // Find the range of the stock in the last N minutes. N is the furthest // back that we will go. If we have some data, but less than N // historical candles, we will use what we have. // // The range is the difference between the highest and the lowest prices // in the given time period. //////////////////////////////////////////////////////////////////// class IntraDayRange : public GenericDataNode { private: ShortTermCandles *_candles; int_fast16_t _candleCount; // Cache the highest and the lowest of the historical candles. // Recompute the current candle each time because it is easier that // way, and we wouldn't be saving much work, anyway. mutable double _highest, _lowest; mutable int _epoch; void updateHistoricalHighLow() const; IntraDayRange(DataNodeArgument const &args); friend class GenericDataNodeFactory; public: void getDouble(bool &valid, double &value) const; }; IntraDayRange::IntraDayRange(DataNodeArgument const &args) : _epoch(0) { DataNodeArgumentVector const &argList = args.getListValue(); assert(argList.size() == 2); // Expected params: (Symbol, Count) const int minutes = argList[1].getIntValue(); int secondsPerCandle; if (minutes <= 2) { // Precise to the second _candleCount = minutes * MARKET_HOURS_MINUTE; secondsPerCandle = 1; } else { // Precise to the minute _candleCount = minutes; secondsPerCandle = MARKET_HOURS_MINUTE; } addAutoLink(ShortTermCandles::find(this, 0, _candles, argList[0].getStringValue(), secondsPerCandle)); } void IntraDayRange::getDouble(bool &valid, double &value) const { _candles->checkHistoryState(); valid = (_candles->getHistoryCount() > 0) || (_candles->currentCandleValid()); if (!valid) return; updateHistoricalHighLow(); if (_candles->currentCandleValid()) value = std::max(_highest, _candles->getCurrentCandle().highPrice) - std::min(_lowest, _candles->getCurrentCandle().lowPrice); else value = _highest - _lowest; valid = value >= 0; } void IntraDayRange::updateHistoricalHighLow() const { // Assume we've already updated the candles: _candles->checkHistoryState(); if (_candles->getEpoch() != _epoch) { _epoch = _candles->getEpoch(); _highest = -std::numeric_limits< double >::max(); _lowest = std::numeric_limits< double >::max(); for (int i = std::min(_candleCount, _candles->getHistoryCount()); i > 0; i--) { SingleCandle const &c = _candles->getCandle(i); if (c.highPrice > _highest) _highest = c.highPrice; if (c.lowPrice < _lowest) _lowest = c.lowPrice; } } } //////////////////////////////////////////////////////////////////// // IntraDayPositionInRange // // First, find the range of the stock in the last N minutes. N is // the furthest back that we will go. If we have some data, but // less than N historical candles, we will use what we have. // // Then compare the current price to that range. This is how we // compute the daily position in range: // (price-t_low)/(t_high-t_low)*100 //////////////////////////////////////////////////////////////////// class IntraDayPositionInRange : public GenericDataNode { private: ShortTermCandles *_candles; int_fast16_t _candleCount; // Cache the highest and the lowest of the historical candles. // Recompute the current candle each time because it is easier that // way, and we wouldn't be saving much work, anyway. mutable double _highest, _lowest; mutable int _epoch; void updateHistoricalHighLow() const; IntraDayPositionInRange(DataNodeArgument const &args); friend class GenericDataNodeFactory; public: void getDouble(bool &valid, double &value) const; }; IntraDayPositionInRange::IntraDayPositionInRange(DataNodeArgument const &args) : _epoch(0) { DataNodeArgumentVector const &argList = args.getListValue(); assert(argList.size() == 2); // Expected params: (Symbol, Count) const int minutes = argList[1].getIntValue(); int secondsPerCandle; if (minutes <= 2) { // Precise to the second _candleCount = minutes * MARKET_HOURS_MINUTE; secondsPerCandle = 1; } else { // Precise to the minute _candleCount = minutes; secondsPerCandle = MARKET_HOURS_MINUTE; } addAutoLink(ShortTermCandles::find(this, 0, _candles, argList[0].getStringValue(), secondsPerCandle)); } void IntraDayPositionInRange::getDouble(bool &valid, double &value) const { _candles->checkHistoryState(); valid = ((_candles->getHistoryCount() > 0) || (_candles->currentCandleValid())) && _candles->tosValid(); if (!valid) return; updateHistoricalHighLow(); double highest; double lowest; if (_candles->currentCandleValid()) { highest = std::max(_highest, _candles->getCurrentCandle().highPrice); lowest = std::min(_lowest, _candles->getCurrentCandle().lowPrice); } else { highest = _highest; lowest = _lowest; } valid = highest > lowest; if (!valid) return; value = (_candles->tosValue().price - lowest) / (highest - lowest) * 100.0; /* else { TclList msg; msg<symbol() <<"range?"<<_highest - _lowest <<"getHistoryCount()"<<_candles->getHistoryCount() <<"currentCandleValid()"<<(_candles->currentCandleValid()?'1':'0') <<"highest"<getCurrentCandle().highPrice <<"lowest"<getCurrentCandle().lowPrice; sendToLogFile(msg); value = -99990; } */ } void IntraDayPositionInRange::updateHistoricalHighLow() const { // Assume we've already updated the candles: _candles->checkHistoryState(); if (_candles->getEpoch() != _epoch) { _epoch = _candles->getEpoch(); _highest = -std::numeric_limits< double >::max(); _lowest = std::numeric_limits< double >::max(); for (int i = std::min(_candleCount, _candles->getHistoryCount()); i > 0; i--) { SingleCandle const &c = _candles->getCandle(i); if (c.highPrice > _highest) _highest = c.highPrice; if (c.lowPrice < _lowest) _lowest = c.lowPrice; } } } //////////////////////////////////////////////////////////////////// // Global //////////////////////////////////////////////////////////////////// void initializeShortTermCandles() { GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement1", symbolPlaceholderObject, 1); GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement2", symbolPlaceholderObject, 2); GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement5", symbolPlaceholderObject, 5); GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement10", symbolPlaceholderObject, 10); GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement15", symbolPlaceholderObject, 15); GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement30", symbolPlaceholderObject, 30); GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement60", symbolPlaceholderObject, 60); GenericDataNodeFactory::sf< TimedMovement > ("TimedMovement120", symbolPlaceholderObject, 120); GenericDataNodeFactory::sf< VolumeChange > ("VolumeChange1", symbolPlaceholderObject, 1); GenericDataNodeFactory::sf< VolumeChange > ("VolumeChange2", symbolPlaceholderObject, 2); GenericDataNodeFactory::sf< VolumeChange > ("VolumeChange5", symbolPlaceholderObject, 5); GenericDataNodeFactory::sf< VolumeChange > ("VolumeChange10", symbolPlaceholderObject, 10); GenericDataNodeFactory::sf< VolumeChange > ("VolumeChange15", symbolPlaceholderObject, 15); GenericDataNodeFactory::sf< VolumeChange > ("VolumeChange30", symbolPlaceholderObject, 30); GenericDataNodeFactory::sf< TimedMovementPercent >("QQQQ5", "QQQ", 5); GenericDataNodeFactory::sf< TimedMovementPercent >("QQQQ10", "QQQ", 10); GenericDataNodeFactory::sf< TimedMovementPercent >("QQQQ15", "QQQ", 15); GenericDataNodeFactory::sf< TimedMovementPercent >("QQQQ30", "QQQ", 30); GenericDataNodeFactory::sf< TimedMovementPercent >("SPY5", "SPY", 5); GenericDataNodeFactory::sf< TimedMovementPercent >("SPY10", "SPY", 10); GenericDataNodeFactory::sf< TimedMovementPercent >("SPY15", "SPY", 15); GenericDataNodeFactory::sf< TimedMovementPercent >("SPY30", "SPY", 30); GenericDataNodeFactory::sf< TimedMovementPercent >("DIA5", "DIA", 5); GenericDataNodeFactory::sf< TimedMovementPercent >("DIA10", "DIA", 10); GenericDataNodeFactory::sf< TimedMovementPercent >("DIA15", "DIA", 15); GenericDataNodeFactory::sf< TimedMovementPercent >("DIA30", "DIA", 30); GenericDataNodeFactory::sf< IntraDayRange > ("IntraDayRange2", symbolPlaceholderObject, 2); GenericDataNodeFactory::sf< IntraDayRange > ("IntraDayRange5", symbolPlaceholderObject, 5); GenericDataNodeFactory::sf< IntraDayRange > ("IntraDayRange15", symbolPlaceholderObject, 15); GenericDataNodeFactory::sf< IntraDayRange > ("IntraDayRange30", symbolPlaceholderObject, 30); GenericDataNodeFactory::sf< IntraDayRange > ("IntraDayRange60", symbolPlaceholderObject, 60); GenericDataNodeFactory::sf< IntraDayRange > ("IntraDayRange120", symbolPlaceholderObject, 120); GenericDataNodeFactory::sf< IntraDayPositionInRange > ("IntraDayPositionInRange5", symbolPlaceholderObject, 5); GenericDataNodeFactory::sf< IntraDayPositionInRange > ("IntraDayPositionInRange15", symbolPlaceholderObject, 15); GenericDataNodeFactory::sf< IntraDayPositionInRange > ("IntraDayPositionInRange30", symbolPlaceholderObject, 30); GenericDataNodeFactory::sf< IntraDayPositionInRange > ("IntraDayPositionInRange60", symbolPlaceholderObject, 60); }