#include "../for_activ/StandardActivHeaders.h"

#include "../../shared/MiscSupport.h"
#include "../../shared/TwoDLookup.h"
#include "../for_activ/ActivSupport.h"
#include "StockInfo.h"

#include "ReadFromActiv.h"

typedef Activ::ContentPlatform::ContentGatewayApi::ContentGatewayClient::GetFirst GetFirst;
typedef Activ::ContentPlatform::ContentGatewayApi::ContentGatewayClient::GetNext GetNext;

static const bool dumpAllMessages = false;

class MyContentGatewayClient : public Activ::ContentPlatform::ContentGatewayApi::ContentGatewayClient
{
private:
  static const Activ::ContentPlatform::Feed::TableNo TABLE_NO = Activ::ContentPlatform::Feed::TABLE_NO_US_LISTING;
  static const int RECORDS_PER_REQUEST = 500;

  // This is a complete dump of what we read from Activ.  It will overwrite some duplicates, but I don't expect that
  // to be a problem.
  TwoDArray _allFieldsDebug;

  // This maps from the activ symbol to the normal version that we use everywhere else.
  std::map< std::string, std::string > _symbolTranslation;

  // Ignore stocks not listed on one of these exchanges.
  std::set< std::string > _exchanges;
  void setExchanges(std::vector< std::string > const &exchanges);

  // This is what we export to the main program.  It might get merged with data from the database.
  StockInfoList _stockInfo;

  Activ::ContentPlatform::ContentGatewayApi::FieldListValidator _fieldListValidator;
  Activ::ContentPlatform::ContentGatewayApi::SymbolId _lastSymbol;
  bool _moreData;

  void doGetFirst();
  void doGetNext();
  void processResponseBlockList(Activ::ContentPlatform::ContentGatewayApi::ResponseBlockList const &responseBlockList);

public:
  MyContentGatewayClient(Activ::Application &application);

  void getSymbols();
  void dumpSymbolTranslation(std::string const &fileName) const;
  StockInfoList getStockInfoList() const { return _stockInfo; }
};

static void dumpFieldData(Activ::ContentPlatform::ContentGatewayApi::SymbolId const &resolvedKey, 
			  Activ::ContentPlatform::ContentGatewayApi::FieldData const &fieldData,
			  Activ::ContentPlatform::ContentGatewayApi::FieldListValidator &fieldListValidator,
			  Activ::ContentPlatform::ContentGatewayApi::ContentGatewayClient const &client,
			  TwoDArray &out)
{ // By using the resolved key, instead of the response key, it's possible that we will overwrite some data.
  // In particular, if the same field exists in more than one table, that will happen.  Symbol is the only field
  // I know of which is like this, so I don't think this is a problem.  Furthermore, when I had one row per
  // response key, I had too many rows and XL would not load the file.
  std::string const &row = resolvedKey.m_symbol;

  if (fieldData.IsEmpty())
    {
      out.add("empty", row, "1");
      return;
    }

  Activ::StatusCode result = fieldListValidator.Initialize(fieldData);
  if (result != Activ::STATUS_CODE_SUCCESS)
    {
      out.add("error", row, Activ::StatusCodeToString(result));
      return;
    }

  for (Activ::ContentPlatform::ContentGatewayApi::FieldListValidator::ConstIterator it = fieldListValidator.Begin(), itEnd = fieldListValidator.End();
       it != itEnd; ++it)
    {
      const Activ::ContentPlatform::ContentGatewayApi::FieldListValidator::Field &field = it.GetField();
      const std::string col = Activ::ContentPlatform::ContentGatewayApi::MetaDataRequestHelper::GetUniversalFieldHelper(client, field.m_fieldId).m_name + " [" + ntoa(field.m_fieldId) + ']';
      const std::string value = 
	(Activ::ContentPlatform::FeedApi::FIELD_STATUS_DEFINED == field.m_fieldStatus)
	? field.m_pIFieldType->ToString()
	: Activ::ContentPlatform::FeedApi::FieldStatusToString(field.m_fieldStatus);
      out.add(col, row, value);
    }
}

void MyContentGatewayClient::setExchanges(std::vector< std::string > const &exchanges)
{
  _exchanges.clear();
  for (std::vector< std::string >::const_iterator it = exchanges.begin();
       it != exchanges.end();
       it++)
    {
      std::string const &exchange = trim(*it);
      if (!exchange.empty())
	_exchanges.insert(exchange);
    }
  if (_exchanges.empty())
    { // Fill in defaults.
      _exchanges.insert("Q");  // NASDAQ
      _exchanges.insert("N");  // New York
      _exchanges.insert("QO"); // NASDAQ OTC
      _exchanges.insert("QB"); // NASDAQ Bulletin Board.
      _exchanges.insert("A");  // AMEX
      _exchanges.insert("PA"); // ArcaExchange
      // PS - pink sheets?
    }
}

void MyContentGatewayClient::dumpSymbolTranslation(std::string const &fileName) const
{ // The Activ form is the key, and the "normal" version is in the column
  // labeled "normal".  In normal use, we look things up both ways, so it
  // doesn't matter which way we store this.  But if we make a mistake, the
  // normal symbol won't be unique.  We assume that the Activ version is
  // always unique.  So this lets someone look for a problem.
  TwoDArray csv;
  static const std::string col = "normal";
  for (std::map< std::string, std::string >::const_iterator it = 
	 _symbolTranslation.begin();
       it != _symbolTranslation.end();
       it++)
    csv.add(col, it->first, it->second);
  csv.writeToCSV(fileName);
}

std::string activToNormal(std::string activ)
{ // This is not complete.  We need to have special extensions for canadian stocks or we will have duplicates.
  std::string::size_type period = activ.find('.');
  assert(period != std::string::npos);
  return activ.substr(0, period);
}

struct ItemInProgress
{
  std::string exchange;
  double dollarVolume;
  time_t closeDate;
  bool valid;
  ItemInProgress() : dollarVolume(0.0), closeDate(0), valid(false) {}
};

void MyContentGatewayClient::processResponseBlockList(Activ::ContentPlatform::ContentGatewayApi::ResponseBlockList const &responseBlockList)
{ // If this response was empty, stop asking for more.
  _moreData = false;

  std::map< std::string, ItemInProgress > itemsInProgress;

  for (Activ::ContentPlatform::ContentGatewayApi::ResponseBlockList::const_iterator responseBlockListIterator =
	 responseBlockList.begin(),
	 responseBlockListIteratorEnd = responseBlockList.end();
       responseBlockListIteratorEnd != responseBlockListIterator; 
       ++responseBlockListIterator)
    {
      if ((responseBlockListIterator->IsValidResponse()) &&
	  (Activ::STATUS_CODE_SUCCESS == _fieldListValidator.Initialize(responseBlockListIterator->m_fieldData)))
	{ // In other Activ calls, these come back in random order.
	  // Here I seem to be getting all the RELATIONSHIP_ID_NONE items first, followed by all the RELATIONSHIP_ID_SECURITY
	  // items.  Within that list, they seem to be sorted by symbol.
	  // The resolved symbol is something like "DELL." or "DELL.Q".  It's a composite symbol, or a regional.
	  // The response key is something like "AAIR.QB-ANL" or "AAIAF.SEC".  These are created as part of the
	  // relationship id.
	  if (_lastSymbol < responseBlockListIterator->m_resolvedKey)
	    // Keep track of the last symbol, so we can start after it next time.
	    _lastSymbol = responseBlockListIterator->m_resolvedKey;
	  Activ::ContentPlatform::ContentGatewayApi::FieldData const &fieldData = responseBlockListIterator->m_fieldData;
	  if (dumpAllMessages)
	    dumpFieldData(responseBlockListIterator->m_resolvedKey, fieldData, _fieldListValidator, *this, _allFieldsDebug);

	  // We received something.  So it's worth looking for more.
	  _moreData = true;

	  ItemInProgress &item = itemsInProgress[responseBlockListIterator->m_resolvedKey.m_symbol];
	  for (Activ::ContentPlatform::ContentGatewayApi::FieldListValidator::ConstIterator it = _fieldListValidator.Begin(), itEnd = _fieldListValidator.End();
	       it != itEnd; ++it)
	    {
	      const Activ::ContentPlatform::ContentGatewayApi::FieldListValidator::Field &field = it.GetField();
	      switch (field.m_fieldId)
		{
		case Activ::ContentPlatform::Feed::FID_SYMBOL:
		  if (responseBlockListIterator->m_resolvedKey.m_symbol + "-ANL" == fieldFromTextString(field))
		    // We only pick this field from the analytics table.  That maps everything based on the main
		    // symbol.  That's a composite if one exists.
		    item.valid = true;
		  break;
		case Activ::ContentPlatform::Feed::FID_CUMULATIVE_VALUE:
		case Activ::ContentPlatform::Feed::FID_PREVIOUS_CUMULATIVE_VALUE:
		  item.dollarVolume = std::max(item.dollarVolume, fieldFromRational(field));
		  break;
		case Activ::ContentPlatform::Feed::FID_CLOSE_DATE:
		  item.closeDate = fieldFromDate(field);
		  break;
		case Activ::ContentPlatform::Feed::FID_PRIMARY_EXCHANGE:
		  item.exchange = fieldFromTextString(field);
		  break;
		}
	    } 
	}
    }
  for (std::map< std::string, ItemInProgress >::const_iterator it = itemsInProgress.begin();
       it != itemsInProgress.end();
       it++)
    if (it->second.valid && _exchanges.count(it->second.exchange))
      {
	StockInfo info;
	info.symbol = activToNormal(it->first);
	info.closeDate = it->second.closeDate;
	info.dollarVolume = it->second.dollarVolume;
	_stockInfo.push_back(info);
	_symbolTranslation[it->first] = info.symbol;
      }
}

/* I found several stocks coming up with no exchange.  One said it was listed on MW.

could not find exchange:  '' AAZST.
could not find exchange:  '' AOMOF.QO
could not find exchange:  '' AZRDF.QO
could not find exchange:  '' BSL.
could not find exchange:  '' DCV.
could not find exchange:  '' ELI.
could not find exchange:  '' GZMIF.QO
could not find exchange:  '' IMM.
could not find exchange:  '' ITU.
could not find exchange:  '' KZBGF.QO
could not find exchange:  '' PDQ.
could not find exchange:  '' PEH.
could not find exchange:  'MW' PEM.
could not find exchange:  '' PFP.
could not find exchange:  '' PGZ.
could not find exchange:  '' PHJ.
could not find exchange:  '' PHW.
could not find exchange:  '' PRY.
could not find exchange:  '' PUA.
could not find exchange:  '' PVM.
could not find exchange:  '' PWD.
could not find exchange:  '' RCHPF.QO
could not find exchange:  '' TESU.QO
could not find exchange:  '' TESW.QO
could not find exchange:  '' ZJZZT.
could not find exchange:  '' ZVZZT.
could not find exchange:  '' ZYSTF.
could not find exchange:  '' ZYSZZ.
could not find exchange:  '' ZYYZZ.
could not find exchange:  '' ZZZZQ.QO

I hope it's safe to ignore these!

ZJZZT and ZVZZT are definately test symbols and should not be in the list.
*/

void MyContentGatewayClient::doGetFirst()
{
  GetFirst::RequestParameters requestParameters;
  requestParameters.m_tableNumber = TABLE_NO;
  requestParameters.m_numberOfRecords = RECORDS_PER_REQUEST;
  Activ::ContentPlatform::ContentGatewayApi::RequestBlock requestBlock;

  requestBlock.m_relationshipId = Activ::ContentPlatform::Feed::RELATIONSHIP_ID_NONE;
  // It seems that cumulative value and previous cumulative value were both valid (and different) on Saturday, but on Sunday
  // only previous cumulative value is valid.  That's not true.  The previous statement was correct for a list of 6.  When I
  // asked for 100 I got something more complicated.  It appears that I cannot expect the results to come back in any
  // particular order.
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_CUMULATIVE_VALUE);
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_PREVIOUS_CUMULATIVE_VALUE);
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_CLOSE_DATE);
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_LOCAL_CODE);
  requestParameters.m_requestBlockList.push_back(requestBlock);
  requestBlock.m_fieldIdList.clear();
  requestBlock.m_relationshipId = Activ::ContentPlatform::Feed::RELATIONSHIP_ID_SECURITY;
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_PRIMARY_EXCHANGE);
  requestParameters.m_requestBlockList.push_back(requestBlock);
  requestBlock.m_fieldIdList.clear();
  requestBlock.m_relationshipId = Activ::ContentPlatform::Feed::RELATIONSHIP_ID_ANALYTICS;
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_SYMBOL);
  requestParameters.m_requestBlockList.push_back(requestBlock);
  GetFirst::ResponseParameters responseParameters;
  Activ::StatusCode statusCode = GetFirst::SendRequest(*this, requestParameters, responseParameters, 500);
  if (Activ::STATUS_CODE_SUCCESS != statusCode)
    {
      // REPORT ERROR
      std::cout << __FUNCTION__ << ": GetFirst::SendRequest() returned " << Activ::StatusCodeToString(statusCode) << std::endl;
      _moreData = false;
      return;
    }
  processResponseBlockList(responseParameters.m_responseBlockList);
}

void MyContentGatewayClient::doGetNext()
{
  static int count = 0;
  count++;
  std::cout<<"Requesting more from Activ ("<<count<<')'<<std::endl;

  GetNext::RequestParameters requestParameters;
  requestParameters.m_numberOfRecords = RECORDS_PER_REQUEST;
  requestParameters.m_symbolId = _lastSymbol;
  Activ::ContentPlatform::ContentGatewayApi::RequestBlock requestBlock;

  requestBlock.m_relationshipId = Activ::ContentPlatform::Feed::RELATIONSHIP_ID_NONE;
  // It seems that cumulative value and previous cumulative value were both valid (and different) on Saturday, but on Sunday
  // only previous cumulative value is valid.  That's not true.  The previous statement was correct for a list of 6.  When I
  // asked for 100 I got something more complicated.  It appears that I cannot expect the results to come back in any
  // particular order.
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_CUMULATIVE_VALUE);
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_PREVIOUS_CUMULATIVE_VALUE);
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_CLOSE_DATE);
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_LOCAL_CODE);
  requestParameters.m_requestBlockList.push_back(requestBlock);
  requestBlock.m_fieldIdList.clear();
  requestBlock.m_relationshipId = Activ::ContentPlatform::Feed::RELATIONSHIP_ID_SECURITY;
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_PRIMARY_EXCHANGE);
  requestParameters.m_requestBlockList.push_back(requestBlock);
  requestBlock.m_fieldIdList.clear();
  requestBlock.m_relationshipId = Activ::ContentPlatform::Feed::RELATIONSHIP_ID_ANALYTICS;
  requestBlock.m_fieldIdList.push_back(Activ::ContentPlatform::Feed::FID_SYMBOL);
  requestParameters.m_requestBlockList.push_back(requestBlock);
  GetFirst::ResponseParameters responseParameters;
  Activ::StatusCode statusCode = GetNext::SendRequest(*this, requestParameters, responseParameters, 500);
  if (Activ::STATUS_CODE_SUCCESS != statusCode)
    {
      // REPORT ERROR
      if (statusCode != Activ::STATUS_CODE_NOT_FOUND)
	std::cout << __FUNCTION__ << ": GetNext::SendRequest() returned " << Activ::StatusCodeToString(statusCode) << std::endl;
      _moreData = false;
      return;
    }
  processResponseBlockList(responseParameters.m_responseBlockList);
}

void MyContentGatewayClient::getSymbols()
{
  doGetFirst();
  while (_moreData)
    doGetNext();
  if (dumpAllMessages)
    _allFieldsDebug.writeToCSV("/tmp/all_activ_fields.csv");
}

MyContentGatewayClient::MyContentGatewayClient
(Activ::Application &application) :
  ContentGatewayClient(application, application)
{ // Set exchange list to the default.
  setExchanges(std::vector< std::string >());

  // Standard connection code.
  const std::string m_network="*";
  const std::string m_service_id="Service.ContentGateway";
  const std::string attributeQuery =
    std::string("(plant=") + m_network + std::string(")");
  Activ::ServiceInstanceList serviceInstanceList;
  Activ::StatusCode result =
    Activ::ServiceApi::FindServices(*this, m_service_id,
                                    Activ::AGENT_SCOPE_GLOBAL,
                                    attributeQuery,
                                    serviceInstanceList);
  if (result != Activ::STATUS_CODE_SUCCESS)
    {
      std::cout<<"ServiceApi::FindServices() ->"
               <<Activ::StatusCodeToString(result)<<"\n";
      return;
    }
  // here we are just going to pick the first service that is returned,
  // and its first access point url
  ContentGatewayClient::ConnectParameters connectParameters;
  const Activ::ServiceInstance &serviceInstance = serviceInstanceList.front();

  connectParameters.m_serviceId = serviceInstance.m_serviceId;
  connectParameters.m_url =
    serviceInstance.m_serviceAccessPointList.front().m_url;
  connectParameters.m_userId = "idea5000-user1";
  connectParameters.m_password = "idea-u1";
  connectParameters.m_policy =
    ConnectParameters::POLICY_FIND_RANDOM_ALTERNATIVE;
  connectParameters.m_attributeQuery = attributeQuery;

  result = ContentGatewayClient::Connect(connectParameters,
                                         Port::defaultTimeout);
  if (result != Activ::STATUS_CODE_SUCCESS)
    {
      std::cout<<"ContentGatewayClient::Connect() -> "
               <<Activ::StatusCodeToString(result)<<"\n";
      return;
    }
  Enable();
}

// I don't know how to destroy the client.  When this was a local variable, 
// and not in dynamic memory, the program would hang when we got to the
// destructor.  So, instead, I don't destroy anything.  I let it live until
// the program exits.
static Activ::AgentApplication *application = NULL;
static MyContentGatewayClient *client = NULL;
static bool previousFailure = false;

// This wasn't really designed to be called more than once.  If I ever need
// to do that, I need to go back through to make sure I restore the state of
// the client before the second time through.
void readFromActiv(std::vector< std::string > const &exchanges, StockInfoList &results)
{
  results.clear();

  if (!application)
    {
      if (previousFailure)
	return;
      Activ::AgentApplication::Settings settings;
      settings.m_enableCtrlHandler = false;
      application = new Activ::AgentApplication(settings);
      client = new MyContentGatewayClient(*application);
      Activ::StatusCode result = application->StartThread();
      if (result != Activ::STATUS_CODE_SUCCESS)
	{ 
	  previousFailure = true;
	  std::cout<<__FILE__<<' '<<__LINE__<<' '<<__FUNCTION__<<" -> "
		   <<Activ::StatusCodeToString(result)<<"\n";
	  return;
	}
    }
  client->getSymbols();
  client->dumpSymbolTranslation("/tmp/SymbolTranslation.csv");
  results = client->getStockInfoList();
}

/*
// This is just a placeholder until we get a real main program.
// The real main program will read from multiple sources, and
// have a lot of processing options to deal with the output 
// from these sources.
int main(int argc, char *argv[])
{
  std::cout<<"STARTING"<<std::endl;
  StockInfoList results;
  readFromActiv(std::vector< std::string >(), results);
  for (StockInfoList::const_iterator it = results.begin();
       it != results.end();
       it++)
    std::cout<<it->symbol<<":  "<<it->closeDate<<"  $"<<ntoa(it->dollarVolume)<<'\n';
  std::cout<<"DONE"<<std::endl;
}
*/

/*  Notes:
Fields:
Symbol 456 -- not sure if we need this.  don't we get this elsewhere?
PreviousCumulativeValue [657] -- Similar to below.  A good backup if the next one fails.
CloseCumulativeValue [70] -- total price * volume.
CloseDate [65] -- report the age to get rid of very old stocks.

PrimaryExchange [341] -- This might be the security relationship.

These functions give us a way to search for a symbol.  You can serach on a few fields, but not many.
You could probably use this with a * to see all stocks, but that doesn't seem like the way to go.
  DoSynchronousGetSymbolsRequest
  void DisplaySymbolDirectoryResponse(GetSymbols::ResponseParameters &responseParameters) const;
  virtual void OnGetSymbolsResponse(HeapMessage &response);

This seems like the example to follow.  Use get first / get next to see all symbols.
  DoSynchronousGetFirstRequest();

This is described in:
  Starts with the middle of page 20 of ActivContentGatewayApi.pdf
  through middle of page 24

The sample code can be found in:
  ~/activ/source/ActivContentPlatform/Samples/ActivFeedSdk/DocumentationSample/MyContentGatewayClient.cpp

*/