Program GetStockList;

{$APPTYPE CONSOLE}

{ This program starts the entire process of generating alerts.  This program
  selects which stock symbols we will be following.

  For a long time this was combined with the program that did the overnight
  processing on the stock symbols (i.e. determining the volatility and
  average volume of each stock).  These two tasks were merged when we switched
  from eSignal to TAL.  By seperating these two tasks again we make things
  simpler and more flexible.

  At the same time as we are splitting up these two tasks, we are adding some
  functionality and fixing a problem.  The problem is that sometimes TAL forgets
  to give us an important stock symbol!  I think this is a relatively recent
  problem, and it's gotten pretty bad.  Solving this problem is the real reason
  why I'm splitting up the code.  As I add more functionality, I risk making
  the program more complicated.  And it was already a mess.  So some cleanup was
  in order.

  The solution to this problem is to look at multiple sources of data.  We still
  ask TAL for a list of all stocks which have traded at least once in the last
  two days.  This is required to find new stocks.  But we also look at our
  database.  We already have a list of all stocks we've been following for the
  last 2-3 weeks, along with the price, average daily volume, and exchange.  So
  we can easily use this additional data in addition to the TAL data, with the
  same logic that we've already been using.

  The exchange is required for a couple of reasons.  For one thing, there is a
  lot of stuff that we just don't follow.  For another thing some servers are
  dedicated to a particular exchange or set of exchanges.

  We use the dollar-volume to determine which stocks are interesting.  First, we
  may only be able to track a certain number of stocks at once.  We always start
  with the stocks with the highest dollar volume; if we need to skip any stocks,
  we skip the ones with the lowest dollar volume.  This was required for the
  production servers a long time ago.  Now the production servers can cover
  all stocks in the markets that we follow.  However, it's still useful for
  testing, since our test servers are never as powerful as our production
  servers.

  We also use the dollar-volume to estimate how active a stock is.  If we split
  the list of stocks into two or more pieces (for two or more servers) we want
  each server to do approximately the same amount of work.  We already have the
  dollar-volume handy, so we use it.  Would volume be better?  The number of
  prints would be ideal, but that would take a little more work, and we have
  does a decent job already.

}

uses
  SysUtils,
  Classes,
  MySqlSupport in '..\..\Misc Util\MySqlSupport.pas',
  mysql in '..\..\Misc Util\mysql.pas',
  ConfigFile in '..\..\Misc Util\ConfigFile.pas',
  SymbolLists in 'SymbolLists.pas',
  Common in 'Common.pas',
  LoadFromDatabase in 'LoadFromDatabase.pas',
  MultiLog in '..\MultiLog.pas',
  LoadFromNxCore in 'LoadFromNxCore.pas',
  PascalNxCoreAPI in '..\..\Shared\PascalNxCoreAPI.PAS',
  StringStringHashtables in '..\..\Shared\StringStringHashtables.pas',
  TwoDLookup in '..\..\Shared\TwoDLookup.pas';

Procedure ShowHelp;
  Begin
    WriteLn(ParamStr(0), ' options');
    WriteLn('--Load-Symbol-File filename:  Load a list of symbols.  May be repeated for multiple files.  These have priority before symbols that came from TAL or the database.');
    WriteLn('--Max-Symbols N:  The total number of symbols will be no more than N.  Symbols from a file are always present.  Symbols from the auto list are ranked and the lowest dollar volume ones are removed.');
    WriteLn('--Skip-NxCore:  Do not load symbols from NxCore.');
    WriteLn('--Load-Tape-File filename:  NxCore tape file for replaying a day.');
    WriteLn('--Live:  This attempts to connect to the current live feed.  This will block forever if NxCoreAccess is not running');
    WriteLn('--Exit-After-Minutes:  This parameter chooses how many minutes after midnight the NxCore tape or live feed should exit.  Default for this program is 210 minutes, or 3:30 AM EST');
    WriteLn('--Skip-Database:  Do not load symbols from the database.');
    WriteLn('--NxCore-Exchange-List-File:  Load a list of exchanges.  These are in the raw forms used by NxCore.  Defaults to all NYSE, NASDAQ, AMEX, and OTC/BB/PINK exg codes');
    WriteLn('--Database-Exchange-List-File:  Load a list of exchanges.  These are in the raw forms used in our database.  Defaults to NYSE, NASDAQ and AMEX.');
    WriteLn('--Save-Symbol-File filename:  Save the resulting set of symbols to this file.  The default is symbols.txt.');     WriteLn('--Help:  This message');
    WriteLn('--Hash-Count N:  Split the symbols into this many distinct partitions.');
    WriteLn('--Hash-First N:  The first partition to use.  Should be at least 0 and less than Hash-Count');
    WriteLn('--Hash-Last N:  The last partition to use.  Defaults to Hash-First  Should be at least 0 and less than Hash-Count');
    WriteLn('--Log-File filename:  Send log to designated file in addition to console.  Defaults to GetStockList.txt');
    WriteLn('--File-Log-Level:  Sets the log level for the file output.  Default is 5.');
    WriteLn('--Console-Log-Level:  Sets the log level for the console output.  Default is 5.');
    WriteLn('--Symbol-Translations-File:  Loads csv file with translations for strangely named symbols');
    WriteLn('  Also allows negative price override for bars in database.');
    WriteLn('Hit Enter to continue.');
    ReadLn;
    Halt(1)
  End;

Function LoadExchangeList(FileName : String) : TStringArray;
  Var
    I : Integer;
    FromFile : TStringList;
  Begin
    FromFile := TStringList.Create;
    Try
      FromFile.LoadFromFile(FileName);
      SetLength(Result, FromFile.Count);
      For I := 0 To Pred(FromFile.Count) Do
        Result[I] := FromFile[I]
    Finally
      FromFile.Free
    End
  End;

Var
  SymbolList : TSymbolList;
  OutputFile : String;

Procedure LoadSymbolFile(Const FileName : String);
  Var
    FileList : TStringList;
    I : Integer;
  Begin  { LoadSymbolFile }
    FileList := TStringList.Create;
    Try
      FileList.LoadFromFile(FileName);
      For I := 0 To Pred(FileList.Count) Do
        SymbolList.AddAndForce(FileList[I])
    Finally
      FileList.Free
    End
  End;  { LoadSymbolFile }

Var
  I : Integer;
  MaxSymbolCount : Integer = MaxInt;
  SkipNxCore : Boolean = False;
  SkipDatabase : Boolean = False;
  NxCoreExchangeList, DatabaseExchangeList : TStringArray;
  SaveSymbolFiles : TStringList;
  TapeName : String;
  HashCount, HashFirst, HashLast : Integer;
  ExitAfterMinutes : Integer;

Begin
  Try
    HashCount := 0;
    HashFirst := -1;
    HashLast := -1;

    SetLength(NxCoreExchangeList, 0);
    SetLength(DatabaseExchangeList, 0);
    SymbolList := TSymbolList.Create;
    SaveSymbolFiles := TStringList.Create;
    OutputFile := 'GetStockList.txt';
    I := 1;
    While I <= ParamCount Do
      Begin
        If ParamStr(I) = '--Load-Symbol-File' Then
          Begin
            Inc(I);
            LoadSymbolFile(ParamStr(I))
          End
        Else If ParamStr(I) = '--Max-Symbols' Then
          Begin
            Inc(I);
            MaxSymbolCount := StrToInt(ParamStr(I))
          End
        Else If ParamStr(I) = '--Skip-NxCore' Then
          SkipNxCore := True
        Else If ParamStr(I) = '--Skip-Database' Then
          SkipDatabase := True
        Else If ParamStr(I) = '--NxCore-Exchange-List-File' Then
          Begin
            Inc(I);
            NxCoreExchangeList := LoadExchangeList(ParamStr(I))
          End
        Else If ParamStr(I) = '--Database-Exchange-List-File' Then
          Begin
            Inc(I);
            DatabaseExchangeList := LoadExchangeList(ParamStr(I))
          End
        Else If ParamStr(I) = '--Save-Symbol-File' Then
          Begin
            Inc(I);
            SaveSymbolFiles.Add(ParamStr(I))
          End
        Else If ParamStr(I) = '--Help' Then
          ShowHelp
        Else If ParamStr(I) = '--Hash-Count' Then
          Begin
            Inc(I);
            HashCount := StrToInt(ParamStr(I))
          End
        Else If ParamStr(I) = '--Hash-First' Then
          Begin
            Inc(I);
            HashFirst := StrToInt(ParamStr(I))
          End
        Else If ParamStr(I) = '--Hash-Last' Then
          Begin
            Inc(I);
            HashLast := StrToInt(ParamStr(I))
          End
                Else If ParamStr(I) = '--Log-File' Then
          Begin
            Inc(I);
            OutputFile := ParamStr(I)
          End
        Else If ParamStr(I) = '--Console-Log-Level' Then
          Begin
            Inc(I);
            SetConsoleLogLevel(StrToInt(ParamStr(I)));
          End
        Else If ParamStr(I) = '--Load-Tape-File' Then
          Begin
            Inc(I);
            TapeName := ParamStr(I)
          End
        Else If ParamStr(I) = '--Live' Then  // Connect to live NxCoreAccess is the default
          TapeName := ''
        Else If ParamStr(I) = '--File-Log-Level' Then
          Begin
            Inc(I);
            SetFileLogLevel(StrToInt(ParamStr(I)));
          End
        Else If ParamStr(I) = '--Exit-After-Minutes' Then
          Begin
            Inc(I);
            ExitAfterMinutes := StrToIntDef(ParamStr(I), 0);
            If ExitAfterMinutes < 1 Then
              ShowHelp
            Else
              SetExitAfterMinutes(ExitAfterMinutes)
          End
      Else If (ParamStr(I) = '--Symbol-Translations-File') And (I < ParamCount) Then
        Begin
          Inc(I);
          LoadSymbolTranslations(ParamStr((I)))
        End
        Else
          Begin
            WriteLn('Unknown option:  ', ParamStr(I));
            ShowHelp
          End;
        Inc(I)
      End;

    InitializeLogFile(OutputFile);

    If Not SkipDatabase Then
      Begin
        If Length(DatabaseExchangeList) = 0 Then
          Begin
            SetLength(DatabaseExchangeList, 4);
            DatabaseExchangeList[0] := 'NYSE';
            DatabaseExchangeList[1] := 'AMEX';
            DatabaseExchangeList[2] := 'NASD';
            DatabaseExchangeList[3] := 'BATS';
          End;
        LoadSymbolsFromDatabase(DatabaseExchangeList, SymbolList)
      End;
    { Load from the database before we load from TAL.  TAL can actively delete
      some items that were in the database. }
    If Not SkipNxCore Then
      Begin
        If Length(NxCoreExchangeList) = 0 Then
          Begin
            SetLength(NxCoreExchangeList, 10);
            NxCoreExchangeList[0] := 'NQEX';
            NxCoreExchangeList[1] := 'NQNM';
            NxCoreExchangeList[2] := 'NYSE';
            NxCoreExchangeList[3] := 'AMEX';
            NxCoreExchangeList[4] := 'PACF';
            NxCoreExchangeList[5] := 'NQBB';
            NxCoreExchangeList[6] := 'NQPK';
            NxCoreExchangeList[7] := 'NQSC';
            NxCoreExchangeList[8] := 'BATS';
            NxCoreExchangeList[9] := 'BATY';
            //NxCoreExchangeList[] := 'TSE';
            //NxCoreExchangeList[] := 'CDNX';

          End;
        LoadSymbolsFromNxCore(TapeName, SymbolList, NxCoreExchangeList)
      End;
    If SaveSymbolFiles.Count = 0 Then
      SaveSymbolFiles.Add('Symbols.txt');
    Assert(HashCount >= 0, 'Invalid Hash-Count');
    If HashCount > 0 Then
      Begin
        // If HashCount = 0 then we don't split things up.  Setting count to 1,
        // and first and last to 0 would have the same effect, but this is more
        // effecient.
        Assert((HashFirst >= 0) And (HashFirst < HashCount), 'Invalid Hash-First');
        If HashLast < 0 Then
          // By default we use just one partition.
          HashLast := HashFirst
        Else
          // But we can use a range of partitions.
          Assert((HashLast >= HashFirst) And (HashLast < HashCount), 'Invalid Hash-Last');
        SymbolList.HashLimit(HashCount, HashFirst, HashLast)
      End;
    SymbolList.SaveToFiles(SaveSymbolFiles, MaxSymbolCount)
  Except
    On E : Exception Do
      Begin
        MultiWriteLn(E.ClassName + ':  ' + E.Message);
        If E Is EExternal Then
          MultiWriteLn('ExceptionAddress:' + IntToHex(Integer((E As EExternal).ExceptionRecord^.ExceptionAddress),8));
        MultiWriteLn('Hit Enter to continue.');
        ReadLn;
        Halt(2)
      End
  End;
  MultiWriteLn('Done...');
End.