#include <fstream>
#include <iostream>
#include <vector>
#include <stdlib.h>
#include <algorithm>
#include <limits>
#include <math.h>
#include <map>
#include <string>
#include "../../shared/GlobalConfigFile.h"
#include "../../shared/MiscSupport.h"
#include "../../shared/DatabaseWithRetry.h"
#include "../../shared/TwoDLookup.h"
#include "GetBarsFromDatabase.h"
#include "SymbolsToDatabase.h"
#include "DataFormats.h"

bool endsWith(std::string const & value, std::string const & ending)
{
  if (ending.size() > value.size()) return false;
  return std::equal(ending.rbegin(), ending.rend(), value.rbegin());
}

double getCutoff(const BarList &bars, double position)
{
  // Precondition:  We know that we have at least 10 candles.
  const int length = bars.size();
  double highest = bars[length - 10].high;
  double lowest = bars[length - 10].low;
  for (int i = length - 9; i <= length - 2; i++)
  {
    highest = std::max(highest, bars[i].high);
    lowest = std::min(lowest, bars[i].low);
  }
  return (highest - lowest) * position + lowest;
}

//topping tail
enum TailType { NONE, TOP, BOTTOM };
TailType tailDirection(const BarList &bars)
{
  const int lastIndex = bars.size() - 1;

    if (bars.size() > 3)
    {
      TailType returnValue = NONE;
      for (int i = lastIndex - 3; i < lastIndex; i++)
      {
        const BarData &bar = bars[i];
        if ((bar.close > bar.open) && (returnValue == NONE || returnValue == TOP))
          returnValue = TOP;
        else if ((bar.open > bar.close) && (returnValue == NONE || returnValue == BOTTOM))
          returnValue = BOTTOM;
        else
          return NONE;
      }
      return returnValue;
    }
    else
      return NONE;
}

struct SectorInfo
{
  StringList symbols;
  std::string name;
};

void loadSymbolFile(const std::string &fileName, StringList &symbols)
{
  std::ifstream inFile;
  std::string symbol;
  inFile.open(fileName.c_str());
  
  if (!inFile) 
  {
    std::cerr << "Unable to open file \"" << fileName << '\"' << std::endl;
    exit(1);
  }
  
  while (inFile >> symbol)
  {
    if (!symbol.empty())
      symbols.push_back(symbol);
  }
  inFile.close();
  std::sort(symbols.begin(), symbols.end());
}

int main(int argc, char *argv[])
{
  StringList allSymbols;
  
  if (argc == 1)
  { // Nothing on the command line.  We could use all defaults, but it seems
    // to make more sense to display help here.
    std::cerr<<"Overnight Symbol Lists Process\n"
             <<"Use the -i / -f syntax to specify values.\n"
             <<"* symbol_files - A comma separated list of symbol files to read.\n"
             <<"                 No default.  At least one filename is required\n"
             <<"* dump_lists_db - Set this to the name of the database to dump lists to\n"
             <<"                   Required.\n"
             <<"* read_bars_db - Set this to the name of the database to read bars from\n"
             <<"                   Required.\n"
             <<"* lists_csv - Set this to the name of the CSV that contains list database mappings.\n"
             <<"* ignore_date - Set this to 1 if you don't care that the stock is up to date, default off.\n"
             <<"* other_overnight_csv - Set this to filename for CSV from GetOtherOvernight.exe, used for\n"
             <<"                        ETFs\n"
             <<"  naics_csv - Used to look up sectors"
	     <<"  naics_user - User to dump sector lists into";
    exit(1);
  }
  
  // default

  if (!addConfigItemsFromCommandLine(argv + 1))
    return 1;
  configItemsComplete();

  
  if (!getConfigItem("symbol_files").empty())
  {
    StringList fileList = explode(",", getConfigItem("symbol_files"));
    for (StringList::const_iterator it = fileList.begin(); it != fileList.end(); it++)
      loadSymbolFile(*it, allSymbols);
  }
  else
  {
    std::cerr << "Parameter symbol_files is required.\n";
    exit(1);
  }
  
  
  std::string dumpDb, barsDb;

  if (!getConfigItem("dump_lists_db").empty())
  {
    dumpDb = getConfigItem("dump_lists_db");
  }
  else
  {
    std::cerr << "Parameter dump_lists_db is required.\n";
    exit(1);
  }
  
  if (!getConfigItem("read_bars_db").empty())
  {
    barsDb = getConfigItem("read_bars_db");
  }
    else
  {
    std::cerr << "Parameter read_bars_db is required.\n";
    exit(1);
  }
 
  TwoDArray otherOvernight;
  if (!getConfigItem("other_overnight_csv").empty())
  {
    otherOvernight.loadFromCSV(getConfigItem("other_overnight_csv"));
  }
  if (!getConfigItem("override_csv").empty())
  {
    otherOvernight.appendFromCSV(getConfigItem("override_csv"));
  }

  TwoDArray naics;
  if (!getConfigItem("naics_csv").empty())
  {
    naics.loadFromCSV(getConfigItem("naics_csv"));
  }

  TwoDArray listMappings;
  if (!getConfigItem("lists_csv").empty())
  {
    listMappings.loadFromCSV(getConfigItem("lists_csv"));
  }
  GetBarsFromDatabase bfd(barsDb);
 

  bool ignore_date = false;
  if (getConfigItem("ignore_date") == "1")
  {
    ignore_date=true;
  }

  int naics_user = strtolDefault(getConfigItem("naics_user"), 0);
  std::map< int, SectorInfo > naics_symbol_map;

  
  // setup NAICS lists
  for(StringList::const_iterator it = naics.getRowHeaders().begin();
      it != naics.getRowHeaders().end();
      it++)
  {
    StringList symbols;
    std::string name = "*Sectors";
    std::string last = "**";
    std::string current = "*";
    bool skip_compressed = false;
    for (unsigned int i = 2 ; i <= it->size(); i++)
    {      
      current = naics.get("Industry", it->substr(0,i));
      // compress names so we don't have repeats in the tree structure
      if (last != current)
      {
        name += '|' + current;
        skip_compressed = false;
      }
      else
      {
        skip_compressed = true;
      }
      last = current;
    }
    // pad code so that it shows up in the correct place in the GUI
    // and convert two codes that have name conflicts
    std::string code = *it;
    if (code == "32" || code == "33")
    {
      code = "31";
    }
    else if (code == "45")
    {
      code = "44";
    }
    else if (code == "49")
    {
      code = "48";
    }
    else if (code == "323110")
    {
      code = "322999";
    }
    else if (code == "334510")
    {
      code = "334499";
    }

    while (code.size() < 6)
    {
      code += '0';
    }
    int list_id = strtolDefault(code, 0);
    if (list_id > 0 && naics_user > 0 && !skip_compressed)
    {
      std::cout << *it << '\t' << name << std::endl;
      SectorInfo sector;
      StringList empty;
      sector.name = name;
      sector.symbols = empty;
      naics_symbol_map[list_id] = sector;
      //symbolsToDatabase.sendSymbols(naics_user, list_id, symbols, name);
    }
  }
  //exit(0);
  
  static const unsigned int WIDTH = 40;
  static const double MIN_HEIGHT = 0.09;
  std::cout << "WIDTH:  " << WIDTH << std::endl;
  std::cout << "MIN_HEIGHT:  " << MIN_HEIGHT << std::endl;
  
  StringList doji;
  StringList dragonflyDoji;
  StringList gravestoneDoji;
  StringList longLeggedDoji;
  StringList hammer;
  StringList hangingMan;
  StringList wideRangeBar;
  StringList bullishEngulfing;
  StringList bearishEngulfing;
  StringList darkCloudCover;
  StringList piercingPattern;
  StringList narrowRangeBuy;
  StringList narrowRangeSell;
  StringList toppingTail;
  StringList bottomingTail;
  StringList greenBarReversal;
  StringList redBarReversal;
  StringList upwardThrust;
  StringList downwardThrust;
  StringList etf;
  StringList etfProshares;
  StringList etfDirexion;
  StringList etfIpath;
  StringList etfIshares;
  StringList etfFactorshares;
  StringList etfPowershares;
  StringList etfSpdr;
  StringList etfHoldr;
  StringList adr;
  StringList warrants;
    
  //process the symbols
  for(StringList::iterator it = allSymbols.begin(); it != allSymbols.end(); it++)
  {
    /* std::cout << *it; ... */
    BarList bars = bfd.adjustPrices(bfd.getBars(*it, ignore_date), bfd.getSplits(*it));
    double chartHeight;
    std::cout << "Processing " <<  *it << " - " << otherOvernight.get("Name", *it) << ':' << std::endl;
    std::cout << "\t# of daily bars:  " << bars.size() << std::endl;
    if (bars.size() < WIDTH)
      chartHeight = 0.0;
    else
    {
      double chartTop = std::numeric_limits< double >::min();
      double chartBottom = std::numeric_limits< double >::max();
      for (unsigned int i = bars.size() - WIDTH; i < bars.size(); i++)
      {
        chartTop = std::max(chartTop, bars[i].high);
        chartBottom = std::min(chartBottom, bars[i].low);
      }
      chartHeight = chartTop - chartBottom;
    }
    std::cout << "\tChart Height:  " << chartHeight << std::endl;  
    if (!bars.empty())
    {
      const int lastIndex = bars.size() - 1;
      BarData const &lastBar = bars.back();
      BarData const &previousBar = bars[bars.size() - 2];
      const bool chartHeightValid = chartHeight > 0.0;
      std::cout << "\tChart Height Valid:  " << chartHeightValid << std::endl;
      const double interestingHeight = chartHeight * MIN_HEIGHT;
      std::cout << "\tInteresting Height:  " << interestingHeight << std::endl;  
      
      // Formulas based on SimpleCandles.C in chartHeightValid
      // Dojis
      if (chartHeightValid)
      {
        if (lastBar.open == lastBar.close && (lastBar.high - lastBar.low > interestingHeight))
        {
          doji.push_back(*it);
          std::cout << "\t\tDoji    - " << lastBar.open << " - " << lastBar.close << " - " << (lastBar.high - lastBar.low > interestingHeight) <<  std::endl;

          if (lastBar.open == lastBar.high) 
          {
            std::cout << "\t\t\t" << "Dragonfly Doji" << std::endl;
            dragonflyDoji.push_back(*it);
          }
          else if (lastBar.open == lastBar.low)
          {
            std::cout << "\t\t\t" << "Gravestone Doji" << std::endl;
            gravestoneDoji.push_back(*it);
          }
          else if (((lastBar.high - lastBar.open) > interestingHeight)
             && ((lastBar.open - lastBar.low) > interestingHeight))
          {
            std::cout << "\t\t\t" << "Long-Legged Doji" << std::endl;
            longLeggedDoji.push_back(*it);
          }
        }
        else
          std::cout << "\t\t\tNo Doji - " << lastBar.open << " - " << lastBar.close << " - " << (lastBar.high - lastBar.low > interestingHeight) << std::endl;
      }
      
      // Hammer or Hanging Man
      if (chartHeightValid)
      {        
        double lastBodyTop;
        double lastBodyBottom;
        if (lastBar.open > lastBar.close)
        {
          lastBodyTop = lastBar.open;
          lastBodyBottom = lastBar.close;
        }
        else
        {
          lastBodyTop = lastBar.close;
          lastBodyBottom = lastBar.open;
        }
        bool valid =
          // There is no top wick. 
	  (lastBodyTop == lastBar.high) &&
          // The size of the bottom wick must be at least twice as big as the body.
          ((lastBodyTop - lastBodyBottom) * 2 <= lastBodyBottom - lastBar.low) &&
          // The size of the body must be at least 2% of the range of the chart.
          (lastBodyTop - lastBodyBottom >= chartHeight * 0.02);
        
        if (valid)
        {          
          double gap;
          double quality;
          // Hammer  
          gap = previousBar.low - lastBodyTop;
          quality = (gap/2 + (lastBodyBottom - lastBar.low)) 
            / chartHeight * 100.0;
          valid = quality > 0;
          if (valid)
            valid = lastBar.high < getCutoff(bars, 0.25);
          if (valid)
          {
            hammer.push_back(*it);
            std::cout << "\t\tHammer" << std::endl;
          }
        
          // Hanging Man
          gap = lastBodyBottom - previousBar.high;
          quality = (gap/2 + (lastBodyBottom - lastBar.low))
            / chartHeight * 100.0 * 2.0 / 3.0;
          valid = quality > 0;
          if (valid)
            valid = lastBar.high > getCutoff(bars, 0.75);
          if (valid)
          {
            hangingMan.push_back(*it);
            std::cout << "\t\tHanging Man" << std::endl;
          }          
        }       
      
      }
      
      // GreenBarReversal
      if (chartHeightValid && fabs(lastBar.close - lastBar.open) > 0.0)
      {        
        bool green = lastBar.close > lastBar.open;
        int highLowCount = 0;
        int i = lastIndex - 1;
        do
        {
          if (green)
          { // These should be going down.
            if (bars[i].high >= bars[i - i].high)
              break;
            if (bars[i].low >= bars[i - 1].low)
              break;
          }
          else
          { // These should be going up.
            if (bars[i].high <= bars[i - i].high)
              break;
            if (bars[i].low <= bars[i - 1].low)
              break;
          }
          highLowCount++;
          i--;
        }
        while (i >= 1);
        int colorCount = 0;
        i = lastIndex - 1;
        do
        {
          BarData const &bar = bars[i];
          if ((green && (bar.close >= bar.open))
              || ((!green) && (bar.close <= bar.open)))
            break;
          colorCount++;
          i--;
        }
        while (i >= 0);
        
        if ((colorCount >= 3) || (highLowCount >= 3))
        {
          // We have a reversal!
          const int quality = std::max(colorCount, highLowCount);
          std::string description;
          if (green)
          {
            description = "Green bar after ";
            description += ntoa(quality);
            if (colorCount >= highLowCount)
            {
              description += " red bars";
              if (colorCount == highLowCount)
                description += " with";
            }
            if (colorCount <= highLowCount)
              description += " lower highs and lower lows";
          }
          else
          {
            description = "Red bar after ";
            description += ntoa(quality);
            if (colorCount >= highLowCount)
            {
              description += " green bars";
              if (colorCount == highLowCount)
                description += " with";
            }
            if (colorCount <= highLowCount)
              description += " higher highs and higher lows";
          }
          //report(description, quality)
          if (green)
            greenBarReversal.push_back(*it);
          else
            redBarReversal.push_back(*it);
          std::cout << "Bar Reversal - " << green << " - " << description << " - " << quality << std::endl;
        }
      }
      
      // Wide Range bar
      // TODO only requires 21 bars
      if (chartHeightValid)
      {
        std::cout << "\t\tWRB - ";
        const double lastRange = lastBar.high - lastBar.low;
        std::cout << lastRange << " - ";
        if (lastRange > 0.0)
        {
          const double topWickSize = 
            lastBar.high - std::max(lastBar.open, lastBar.close);
          std::cout << topWickSize << " - ";
          if (topWickSize > 0.15 * lastRange)
          {
            const double bottomWickSize =
              std::min(lastBar.open, lastBar.close) - lastBar.low;
            std::cout << bottomWickSize << " - ";  
            if (bottomWickSize > 0.15 * lastRange)
            {
              double averageRange = 0.0;
              for (int i = lastIndex - 20; i < lastIndex; i++)
                averageRange += bars[i].high - bars[i].low;
              averageRange /= 20.0;
              std::cout << averageRange;                
              if (lastRange > averageRange * 1.5)
              {
                wideRangeBar.push_back(*it);
                std::cout << " - Yes!";
              }
              else
              {
                std::cout << " - No!";
              }
            }
          }
        }
        std::cout << std::endl;
      }
      
      // engulfing
      if (chartHeightValid)
      {
        static const double MIN_FIRST_HEIGHT_RATIO = 0.025;
        static const double MIN_DIFFERENCE_RATIO = 0.01;

        double lastBodyTop, lastBodyBottom, previousBodyTop, previousBodyBottom;
        if (lastBar.close != lastBar.open)
        {
          const bool up = lastBar.close > lastBar.open;
          if (up)
          {
            lastBodyTop = lastBar.close;
            lastBodyBottom = lastBar.open;
            previousBodyTop = previousBar.open;
            previousBodyBottom = previousBar.close;
          }
          else
          {
            lastBodyTop = lastBar.open;
            lastBodyBottom = lastBar.close;
            previousBodyTop = previousBar.close;
            previousBodyBottom = previousBar.open;
          }
          const double minFirstHeight = 
            chartHeight  * MIN_FIRST_HEIGHT_RATIO;
          const double minDifference = 
            chartHeight * MIN_DIFFERENCE_RATIO;
          if ((previousBodyTop - previousBodyBottom > minFirstHeight)
              && (lastBodyTop - previousBodyTop > minDifference)
              && (previousBodyBottom - lastBodyBottom > minDifference))
          {
            double quality = 
              (lastBodyTop - lastBodyBottom) / chartHeight * 100.0;
            int64_t recentVolume = 0;
            for (unsigned int i = bars.size() - 10; i < bars.size() - 1; i++)
              recentVolume += bars[i].volume;
            if (recentVolume > 0)
              quality = std::min(100.0, quality * 0.5 + lastBar.volume * 100.0 / recentVolume);
            
            if (up)
              bullishEngulfing.push_back(*it);
            else
              bearishEngulfing.push_back(*it);
            std::cout << "\t\tEngulfing - " << up << " - " << quality << std::endl;
          }
        }
      }
      
      // dark cloud cover or piercing 
      if (chartHeightValid)
      {
        if (lastBar.close != lastBar.open)
        {
          const bool up = lastBar.close > lastBar.open;
          double lastBodyTop, lastBodyBottom, previousBarHigh, previousBodyTop,
          previousBodyBottom;
          if (up)
          { // Flip the image up side down so that we can use the Dark Cloud
            // Covering definition from above. 
            lastBodyTop = -lastBar.open;
            lastBodyBottom = -lastBar.close;
            previousBodyTop = -previousBar.close;
            previousBodyBottom = -previousBar.open;
            previousBarHigh = -previousBar.low;
          }
          else
          { // This is really a Dark Cloud Covering.
            lastBodyTop = lastBar.open;
            lastBodyBottom = lastBar.close;
            previousBodyTop = previousBar.close;
            previousBodyBottom = previousBar.open;
            previousBarHigh = previousBar.high;
          }
          const double previousBodyMiddle = 
            (previousBodyTop + previousBodyBottom) / 2.0;
          if ((previousBodyTop - previousBodyBottom 
                > chartHeight * 0.04) 
              && (lastBodyTop > previousBarHigh)
              && (lastBodyTop - previousBodyTop > chartHeight * 0.005)
              && (previousBodyMiddle > lastBodyBottom)
              && (lastBodyBottom > previousBodyBottom))
          {
            double quality = 
              ((lastBodyTop - lastBodyBottom) 
              + (previousBodyTop - previousBodyBottom)) 
              / chartHeight * 50.0;
            int64_t recentVolume = 0;
            for (unsigned int i = bars.size() - 20; i < bars.size() - 1; i++)
              recentVolume += bars[i].volume;
            if (recentVolume > 0)
            quality = std::min(100.0, quality * 0.5 + (lastBar.volume + previousBar.volume) * 100.0 / recentVolume);
            if (up)
              piercingPattern.push_back(*it);
            else
              darkCloudCover.push_back(*it);
            std::cout << "\t\tPiercing or Dark - " << up << " - " << quality << std::endl;
          }
        }
      }
      
      // narrow range bars
      if (chartHeightValid)
      {
        if (lastBar.close != lastBar.open)
        {
          const bool buy = lastBar.close > lastBar.open;
          double bodyHeight[3];
          bool failure = false;
         
          for (int i = 0; i < 3; i++)
          {
            BarData const &bar = bars[lastIndex - 1 - i];
            if (buy)
            // Looking for a green bar.
              bodyHeight[i] = bar.close - bar.open;
            else
            // Looking for a red bar.
              bodyHeight[i] = bar.open - bar.close;
            if (bodyHeight[i] <= 0)
              failure = true;
          }
          if (!failure)
          {
            double previousHeights = 0;
            for (int i = 1; i <= 5; i++)
            {
              BarData const &bar = bars[lastIndex - i];
              previousHeights += bar.high - bar.low;
            }
            const double lastHeight = bars[lastIndex].high - bars[lastIndex].low;
            if (lastHeight * 20 >= previousHeights)
              failure = true;
            // Quality can be anywhere between 0 and 100.
            // The last bar contributes 0-20 based on how small it is.  If the
            // high and low are the same, we get 20.  If the size is the maximum
            // allowed to trigger the alert, we get 0.
            // Then we look at the body height of the previous three bars.
            // They should each be red or green depending on the direction of the
            // alert.  If the body height is the entire height of the graph then
            // we call the bar "very" red or green and we add 10 for that
            // bar.  As the body height goes closer to 0, the contribution to
            // quality approaches 0.
            // Finally we look at the total height of the last 5 bars
            // preceeding the most recent bar.  Each of these can contribute
            // up to 10 each to the quality.  If the height of the bar is the
            // entire height of the chart then the bar makes the full
            // contribution.
            if (!failure)
            {
              const double quality = 
                (previousHeights + bodyHeight[0] + bodyHeight[1] + bodyHeight[2]) * 10 
                / chartHeight
                + 20 - lastHeight / previousHeights * 400;
              
              if (buy)
                narrowRangeBuy.push_back(*it);
              else
                narrowRangeSell.push_back(*it);
              std::cout << "\t\tNarrow Range Bar - " << buy << " - " << quality << std::endl;
            }
          }            
        }
      }
      // topping/bottoming tail
      if (chartHeightValid)
      {
        TailType direction = tailDirection(bars);
        if (direction != NONE)
        {
          const double lastBodySize = fabs(lastBar.close - lastBar.open);
          double lastWickSize;
          if (direction == TOP)
            lastWickSize = 
              lastBar.high - std::max(lastBar.open, lastBar.close);
          else
            lastWickSize =
              std::min(lastBar.open, lastBar.close) - lastBar.low;
          const double lastBarSize = lastBar.high - lastBar.low;
          const double prevBarSize = previousBar.high - previousBar.low;
          if ((lastBarSize >= prevBarSize * 0.5)
              && (lastWickSize >= lastBodySize * 0.6)
              && (lastWickSize >= lastBarSize * 0.3333333))
          { // Quality can be anywhere between 0 and 100.  Each of the
            // last four bars contributes 0-25.  For the most recent
            // bar we look at the height of the tail relative to the
            // height of the entire chart.  For the three previous bars
            // we look at the height of the entire bar, low to high.
            double quality = lastWickSize + prevBarSize;
            quality += bars[lastIndex - 2].high - bars[lastIndex - 2].low;
            quality += bars[lastIndex - 3].high - bars[lastIndex - 3].low;
            quality = quality / chartHeight * 25.0;
            
            if (direction == TOP)
              toppingTail.push_back(*it);
            else
              bottomingTail.push_back(*it);
            std::cout << "\t\tTop/Bottom Tail - " << direction << " - " << quality << std::endl;
          }
        }
      }
      // upward/downward thrust
      // We decided we don't really like these and they'te not live
      if (chartHeightValid)
      {
        bool upward = true;
        for (int i = lastIndex - 4; i <= lastIndex; i++)
        {
          if (bars[i].close <= bars[i - 8].close)
          {
            upward = false;
            break;
          }
          if (bars[i].close <= bars[i - 20].close)
          {
            upward = false;
            break;
          }
        }
        bool downward = true;
        for (int i = lastIndex - 4; i <= lastIndex; i++)
        {
          if (bars[i].close >= bars[i - 8].close)
          {
            downward = false;
            break;
          }
          if (bars[i].close >= bars[i - 20].close)
          {
            downward = false;
            break;
          }
        }
        if (upward)
        {
          upwardThrust.push_back(*it);
          std::cout << "\t\tUpward Thrust" << std::endl;
        }
        else if (downward)
        {
          downwardThrust.push_back(*it);
          std::cout << "\t\tDownward Thrust" << std::endl;
        }  
      }
    }

    //ETFs
    if (otherOvernight.get("ETF", *it) == "1")
    {
      etf.push_back(*it);
      std::cout << "\t\tETF" << std::endl;
    }
    
    if (otherOvernight.get("ETF_PROSHARES", *it) == "1")
    {
      etfProshares.push_back(*it);
      std::cout << "\t\tETF PROSHARES" << std::endl;
    }
    else if (otherOvernight.get("ETF_DIREXION", *it) == "1")
    {
      etfDirexion.push_back(*it);
      std::cout << "\t\tETF DIREXION" << std::endl;
    }
    else if (otherOvernight.get("ETF_IPATH", *it) == "1")
    {
      etfIpath.push_back(*it);
      std::cout << "\t\tETF IPATH" << std::endl;
    }
    else if (otherOvernight.get("ETF_ISHARES", *it) == "1")
    {
      etfIshares.push_back(*it);
      std::cout << "\t\tETF ISHARES" << std::endl;
    }
    else if (otherOvernight.get("ETF_FACTORSHARES", *it) == "1")
    {
      etfFactorshares.push_back(*it);
      std::cout << "\t\tETF FACTORSHARES" << std::endl;
    }
    else if (otherOvernight.get("ETF_POWERSHARES", *it) == "1")
    {
      etfPowershares.push_back(*it);
      std::cout << "\t\tETF POWERSHARES" << std::endl;
    }
    else if (otherOvernight.get("ETF_SPDR", *it) == "1")
    {
      etfSpdr.push_back(*it);
      std::cout << "\t\tETF SPDR" << std::endl;
    }
    else if (otherOvernight.get("ETF_HOLDR", *it) == "1")
    {
      etfHoldr.push_back(*it);
      std::cout << "\t\tETF HOLDR" << std::endl;
    }
    else if (otherOvernight.get("ADR", *it) == "1")
    {
      adr.push_back(*it);
      std::cout <<"\t\tADR" << std::endl;
    }
    
    // Warrants
    if (endsWith(otherOvernight.get("Name", *it),"WTS"))
    {
      warrants.push_back(*it);
      std::cout <<"\t\tWarrant" << std::endl;
    }

    // Sectors
    for (unsigned int i = 2 ; i <= 6; i++)
    {          
      std::string code = otherOvernight.get("NAICS", *it);
      std::string subCode = code.substr(0,i);

      // Here we have some exceptions...
      //
      // The two digit codes refer to top level sectors that span multiple
      // so we condense them down to a single code so that all symbols are
      // together at this level
      //
      // The two longer codes conflict with the naming of the shorter
      // versions.  Usually if something ends with a 0, the name is the
      // exact same as the shorter version.  But in categories with 10
      // desecendents this fails.  In the 2007 NAICS codes there are two
      // codes which qualify.  So for the purposes of not having the two
      // resulting lists collide we conver the codes into unused codes
      // when we store them.  The client display logic will still group
      // them correctly even though they are now slightly out of order.
      if (subCode == "32" || subCode == "33")
      {
        subCode = "31";
      }
      else if (subCode == "45")
      {
        subCode = "44";
      }
      else if (subCode == "49")
      {
        subCode = "48";
      }
      else if (subCode == "323110")
      {
	subCode = "322999";
      }      
      else if (subCode == "334510")
      {
       subCode = "334499";
      }

      while (subCode.size() < 6)
      {
        subCode += '0';
      }
     
      int list_id = strtolDefault(subCode, 0);

      if (naics_symbol_map.find(list_id) != naics_symbol_map.end())
      {
        naics_symbol_map[list_id].symbols.push_back(*it);
        //std::cout << *it << " is going into a sector \"" << naics_symbol_map[list_id].name << "\" (" << subCode << ')' << std::endl;
      }
    }

  }
  

  SymbolsToDatabase symbolsToDatabase(dumpDb);
  for (std::map< int, SectorInfo >::iterator it = naics_symbol_map.begin();
       it != naics_symbol_map.end();
       it++)
  {
    if (it->second.symbols.size() > 0)
    {
      symbolsToDatabase.sendSymbols(naics_user, it->first, it->second.symbols, it->second.name);
    }
  }
  
  for(StringList::const_iterator it = listMappings.getRowHeaders().begin();
      it != listMappings.getRowHeaders().end();
      it++)
  {
    int userId = strtolDefault(listMappings.get("user_id", *it), -1);
    int listId =  strtolDefault(listMappings.get("list_id", *it), -1);
    if (userId > 0 && listId > 0)
    {
      std::string strategy = listMappings.get("strategy", *it);
      std::string name = listMappings.get("name", *it);
      std::string defaultName;
      StringList symbols;
      bool valid = true;
      if(strategy == "doji")
      {
        symbols = doji;
        defaultName = "Daily|Doji";
      }
      else if(strategy == "dragonfly_doji")
      {
        symbols = dragonflyDoji;
        defaultName = "Daily|Dragonfly Doji";
      }
      else if(strategy == "gravestone_doji")
      {
        symbols = gravestoneDoji;
        defaultName = "Daily|Gravestone Doji";
      }
      else if(strategy == "long_legged_doji")
      {
        symbols = longLeggedDoji;
        defaultName = "Daily|Long-Legged Doji";
      }
      else if(strategy == "hammer")
      {
        symbols = hammer;
        defaultName = "Daily|Hammer";
      }
      else if(strategy == "hanging_man")
      {
        symbols = hangingMan;
        defaultName = "Daily|Hanging Man";
      }
      else if(strategy == "wide_range_bar")
      {
        symbols = wideRangeBar;
        defaultName = "Daily|Wide Range Bar";
      }
      else if(strategy == "bullish_engulfing")
      {
        symbols = bullishEngulfing;
        defaultName = "Daily|Bullish Engulfing";
      }
      else if(strategy == "bearish_engulfing")
      {
        symbols = bearishEngulfing;
        defaultName = "Daily|Bearish Engulfing";
      }
      else if(strategy == "piercing_pattern")
      {
        symbols = piercingPattern;
        defaultName = "Daily|Piercing Pattern";
      }
      else if(strategy == "dark_cloud_cover")
      {
        symbols = darkCloudCover;
        defaultName = "Daily|Dark Cloud Cover";
      }
      else if(strategy == "narrow_range_buy")
      {
        symbols = narrowRangeBuy;
        defaultName = "Daily|Narrow Range Buy Bar";
      }
      else if(strategy == "narrow_range_sell")
      {
        symbols = narrowRangeSell;
        defaultName = "Daily|Narrow Range Sell Bar";
      }
      else if(strategy == "topping_tail")
      {
        symbols = toppingTail;
        defaultName =  "Daily|Topping Tail";
      }
      else if(strategy == "bottoming_tail")
      {
        symbols = bottomingTail;
        defaultName = "Daily|Bottoming Tail";
      }
      else if(strategy == "green_bar_reversal")
      {
        symbols = greenBarReversal;
        defaultName = "Daily|Green Bar Reversal";
      }
      else if(strategy == "red_bar_reversal")
      {
        symbols = redBarReversal;
        defaultName = "Daily|Red Bar Reversal";
      }
      else if(strategy == "upward_thrust")
      {
        symbols = upwardThrust;
        defaultName = "Daily|Upward Thrust";
      }
      else if(strategy == "downward_thrust")
      {
        symbols = downwardThrust;
        defaultName = "Daily|Downward Thrust";
      }
      else if(strategy == "etf")
      {
        symbols = etf;
        defaultName = "*ETFs";
      }
      else if(strategy == "etf_proshares")
      {
        symbols = etfProshares;
        defaultName = "*ETFs|ProShares";
      }
      else if(strategy == "etf_direxion")
      {
        symbols = etfDirexion;
        defaultName = "*ETFs|Direxion";
      }
      else if(strategy == "etf_ipath")
      {
        symbols = etfIpath;
        defaultName = "*ETFs|iPath";
      }
      else if(strategy == "etf_ishares")
      {
        symbols = etfIshares;
        defaultName = "*ETFs|iShares";
      }
      else if(strategy == "etf_factorshares")
      {
        symbols = etfFactorshares;
        defaultName = "*ETFs|FactorShares";
      }
      else if(strategy == "etf_powershares")
      {
        symbols = etfPowershares;
        defaultName = "*ETFs|PowerShares";
      }
      else if(strategy == "etf_spdr")
      {
        symbols = etfSpdr;
        defaultName = "*ETFs|SPDR";
      }
      else if(strategy == "etf_holdr")
      {
        symbols = etfHoldr;
        defaultName = "*ETFs|HOLDR";
      }
      else if(strategy == "adr")
      {
	symbols = adr;
	defaultName = "ADRs";
      }
      else if (strategy == "warrants")
      {
        symbols =  warrants;
        defaultName = "Warrants";
      }
      else
      {
        valid = false;
      }
 
      if (valid)
      {
        if (name.empty())
        {
          name = defaultName;
        }
        symbolsToDatabase.sendSymbols(userId, listId, symbols, name);
        std::cout << userId << ' ' << listId  << "\t" << symbols.size() << " symbols\t" << name << std::endl;
      }
      else
      {
        std::cout << "Strategy \"" << strategy << "\" doesn't exist!\n";
      }
      
      
    }
    
  }
    
}