#include <iostream>
#include <memory>
#include <zlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <unordered_set>

#include "../shared/Random.h"
#include "../shared/CommandDispatcher.h"
#include "../shared/ThreadMonitor.h"
#include "../shared/DatabaseForThread.h"
#include "../shared/LogFile.h"
#include "../shared/ReplyToClient.h"
#include "../shared/XmlSupport.h"
#include "../shared/ContainerThread.h"
#include "../shared/NewWorkerCluster.h"
#include "../shared/GlobalConfigFile.h"
#include "../ax_alert_server/CloudSupport.h"
#include "../ax_alert_server/FormatTime.h"
#include "../oddsmaker/UserInfo.h"
#include "NfsHelper.h"

#include "MiscNFS.h"


/* This handles various requests that deal with the file system and the
 * database.  It could work with any files, but we expect this to work running
 * on an NFS drive.  The NFS requests should be fast most of the time, but they
 * can fail in a different way.  We keep this separated so if we do run into
 * an NFS problem we don't take down unrelated parts of the system.
 *
 * This is initially aimed at the new cloud layout code.  We're moving some
 * of the big data out of the MySql database and onto a normal NFS drive.  The
 * database isn't optimized for this and sometimes it works better than others.
 * However, if this works well we might do similar things with other data.
 * That's why I call this "Misc" NFS.
 *
 * I broke this into two classes, one for the master database and one for the
 * slaves.  We haven't discussed strict rules for what requests stay in order
 * for the cloud layout code.  As I general rule write requests should probably
 * all be done in one thread, to maintain order, and read requests should all
 * be done in a thread pool for maximum scalability.  See the symbol list code
 * for an alternative, where all read and write requests are guaranteed to be
 * processed in order.
 */

using namespace NfsHelper;

namespace MiscNFS
{
  // Pointer containing an error message
  std::unique_ptr<std::string> strerror {new std::string};
  // Tries to read the entire contents of the named file.
  // On success returns true and sets result to the contents of the file.
  // Otherwise returns false, sets result to "", and errorString() will contain
  // more information.
  //
  // TODO this was mostly copied from ../shared/XmlReader.h, with another
  // similar version available in ../stocktwits/MemoryLeakTest.C.  This version
  // below seems to be pretty general purpose.  It should probably be moved
  // to ../shared/MiscSupport.[Ch].
  bool loadFromFile(std::string const &name, std::string &result)
  { // Inspired by
    // http://insanecoding.blogspot.in/2011/11/how-to-read-in-file-in-c.html
    result.clear();
    std::ifstream in(name, std::ios::in | std::ios::binary);
    in.seekg(0, std::ios::end);
    if (in)
    {
      // If in is invalid (e.g. file not found) and we didn't check for that,
      // tellg() would return -1 and resize() would abort the program.  (If you
      // call seekg(), tellg(), read(), etc., on an invalid stream, by default
      // that's just silently ignored.)
      result.resize(in.tellg());
      in.seekg(0, std::ios::beg);
      in.read(&result[0], result.size());
    }
    if (in)
      return true;
    else
    {
      result.clear();
      return false;
    }
  }

  // Generally speaking there's an implied contract.  If the client sends a
  // request and expects exactly one response, we send exactly one response.
  // It would be tempting to just ignore requests when we are not logged in,
  // but that could lead to a memory leak (or a hanging "Please Wait" window)
  // in the client.
  static void sendNullResponse(ExternalRequest const &request)
  {
    const ExternalRequest::MessageId messageId =
      request.getResponseMessageId();
    if (messageId.present())
      addToOutputQueue(request.getSocketInfo(),
                       XmlNode().asString("API"),
                       messageId);
  }

  // Very similar to the shell command of the same name.
  // Try to open the file.  Fail if we don't have read and write access.
  // If the file doesn't exist, try to create it.  Fail if the directories don't exist or we don't have permissions.
  // Return the empty string on success.
  // On failure return a helpful and user friendly string.
  static std::string touch(std::string const &name)
  {
    int handle = open(name.c_str(), O_RDWR | O_CREAT, PERMISSIONS);
    if (handle < 0){
      *strerror = "System failure: unable to open file:  " + name;
      std::cout<< *strerror <<std::endl;
      return *strerror;
    }
    int error = close(handle);
    if (error)
      // This seems unlikely.  But it's good to check, especially with NFS.
      return errorString();
    return "";  // Success!
  }

  // If all looks good, return true.  Either way we output test info to the log file.
  static bool testPaths()
  {
    std::map< int, std::string > names;
    for (int i = 1; i <= DIVISOR; i++)
    {
      const std::string name = pathForFile(i);
      assert(!name.empty());
      names[i] = name;
    }
    for (auto const &kvp : names)
    {
      const std::string errorMessage = touch(kvp.second);
      if (!errorMessage.empty())
      {
        TclList msg;
        msg<<FLF<<"FAILED"<<names<<kvp.first<<kvp.second<<errorMessage;
        LogFile::primary().sendString(msg);
        return false;
      }
    }
    TclList msg;
    msg<<FLF<<"success"<<names;
    LogFile::primary().sendString(msg);
    return true;
  }

  static const std::string s_mtSaveToCloud = "mtSaveToCloud";
  static const std::string s_mtCloudDelete = "mtCloudDelete";
  static const std::string s_mtCloudRevoke = "mtCloudRevoke";

  class NfsMasterDbThread : public ForeverThreadUser
  {
  private:
    enum { mtSaveToCloud, mtCloudDelete, mtCloudRevoke };

    DatabaseWithRetry &database() const
    { return *DatabaseForThread(DatabaseWithRetry::MASTER); }

    void saveToCloud(ExternalRequest &request, UserId userId)
    {
      ThreadMonitor::SetState tm(s_mtSaveToCloud);
      tm.increment(s_mtSaveToCloud);

      const std::string layout = request.getProperty("layout");

      int originalSize = layout.length();
      std::string toSave(originalSize * 3 / 4, '\x00');
      uint64_t size = toSave.length();
      int compressionResult =
        compress2(reinterpret_cast<unsigned char *>(&toSave[0]), &size,
                  reinterpret_cast<const unsigned char *>(&layout[0]),
                  layout.length(), Z_BEST_COMPRESSION);
      if (compressionResult == Z_OK)
      {
        assert(size <= toSave.length());
        if (size > 655360)
        { // Too big.  We have no direct way to report an error.  But this will
          // have a reasonable result.  When someone tries to read this, the
          // client will see the 0 length result and will display an error
          // message.
          //
          // Note:  65536 came from MySQL.  The file system doesn't have that
          // limitation.  I multiplied that by 10 somewhat arbitrarily.  Things
          // can be bigger now.  But we still have a limit.
          toSave.clear();
          originalSize = 0;
        }
        else
          toSave.resize(size);
      }
      else
      { // Problem compressing it.  Probably it was too small to begin with so
        // compression didn't (significantly) help.
        toSave = layout;
        originalSize = 0;
      }
      std::vector< std::string > sql;
      sql.push_back("INSERT INTO cloud_layout "
                    "SET user_id=" + ntoa(userId) + ", "
                    "share_code=MD5(CONCAT(" + ntoa(getRandom31()) + ",NOW(),"
                    + ntoa(userId) + ")), "
                    "original_size=" + ntoa(originalSize) + ", "
                    "icon='"
                    + mysqlEscapeString(request.getProperty("icon")) + "', "
                    "fill_screen='"
                    + (request.getProperty("fill_screen", false)?'Y':'N')
                    + "', "
                    "clear_previous_layout='"
                    + (request.getProperty("clear_previous_layout", false)
                       ?'Y':'N') + "', "
                    "short_description='"
                    + mysqlEscapeString(request.getProperty("short_description"))
                    + "', "
                    "long_description='"
                    + mysqlEscapeString(request.getProperty("long_description"))
                    + "', "
                    "window_count='"
                    // This should be a number, but we don't trust it because it
                    // came from the client, so put it in quotes and let MySQL
                    // sanitize it.
                    + mysqlEscapeString(request.getProperty("window_count"))
                    + "', "
                    "creation=NOW()");
      sql.push_back("SELECT share_code, strategy_id "
                    "FROM cloud_layout "
                    "WHERE strategy_id=LAST_INSERT_ID()");
      MysqlResultRef result =
        database().tryAllUntilSuccess(sql.begin(), sql.end(),
                                      s_mtSaveToCloud)[1];
      const std::string saveError =
        saveToDisk(result->getIntegerField(1, 0), toSave);
      if (!saveError.empty())
        LogFile::primary().sendString(saveError, request.getSocketInfo());
      XmlNode response;
      response.properties["CODE"] = CLOUD_PREFIX + result->getStringField(0);
      addToOutputQueue(request.getSocketInfo(),
                       response.asString("API"),
                       request.getResponseMessageId());
    }

    void cloudDelete(ExternalRequest &request, UserId userId)
    {
      ThreadMonitor::SetState tm(s_mtCloudDelete);
      tm.increment(s_mtCloudDelete);

      std::string strategyId = request.getProperty("id");
      std::string fileName = pathForFile(strategyId);
      if (!fileName.empty())
      {
        std::vector< std::string > sql;
        const std::string where = "WHERE strategy_id='"
          + mysqlEscapeString(strategyId) + "' AND user_id=" + ntoa(userId);
        sql.push_back("SELECT 1 FROM cloud_layout " + where);
        sql.push_back("DELETE FROM cloud_layout " + where);
        const auto results =
          database().tryAllUntilSuccess(sql.begin(), sql.end(), s_mtCloudDelete);
        if (results[0]->rowIsValid())
        { // We deleted something.  That means that the database was aware of
          // this layout and, more importantly, that the layout belongs to this
          // user.  I.e. this user has permissions to delete this!  The file
          // system doesn't know anything about users, so we have to check
          // the database before trying to delete something from the file
          // system.
          int unlinkError = unlink(fileName.c_str());
          if (unlinkError)
          {
            TclList msg;
            msg<<FLF<<"CAN'T unlink() file"<<fileName;
            LogFile::primary().sendString(msg, request.getSocketInfo());
          }
        }
      }
      // Usually we call sendNullResponse() for errors.  In this case nothing
      // is wrong.  We just don't have anything interesting to say.  The client
      // will want this response so it knows we received the message and it
      // doesn't have to repeat the message.
      sendNullResponse(request);
    }

    void cloudRevoke(ExternalRequest &request, UserId userId)
    { // Note that this does not involve the file system at all.  We purposely
      // chose file names based on the strategy_id field, which does not
      // change, rather than this share code.
      ThreadMonitor::SetState tm(s_mtCloudRevoke);
      tm.increment(s_mtCloudRevoke);
      std::string sql =
        "UPDATE cloud_layout "
        "SET share_code=MD5(CONCAT(share_code,NOW(),"
        + ntoa(getRandom31()) + ",strategy_id)) "
        "WHERE user_id=" + ntoa(userId) + " AND strategy_id='"
        // Again, make sure to quote this value because it came directly from
        // the user.  In a previous version of this code (in SymbolLists.C) we
        // converted this to a number and back, an alternative way to prevent
        // an injection attack.
        + mysqlEscapeString(request.getProperty("id")) + "'";
      database().tryQueryUntilSuccess(sql, s_mtCloudRevoke);
      addToOutputQueue(request.getSocketInfo(),
                       XmlNode().asString("API"),
                       request.getResponseMessageId());
      sendNullResponse(request);  // Confirmation
    }

    virtual void handleRequestInThread(Request *current)
    {
      ExternalRequest &request = dynamic_cast< ExternalRequest & >(*current);
      const UserInfoExport userInfo = userInfoGetInfo(request.getSocketInfo());
      if (userInfo.status != sFull)
        // You must be logged in to save or change anything.
        sendNullResponse(request);
      else
        switch (current->callbackId)
        {
        case mtSaveToCloud:
          saveToCloud(request, userInfo.userId);
          break;
        case mtCloudDelete:
          cloudDelete(request, userInfo.userId);
          break;
        case mtCloudRevoke:
          cloudRevoke(request, userInfo.userId);
          break;
        default:
          abort();
        }
    }

    NfsMasterDbThread() :
      ForeverThreadUser(IContainerThread::create("NfsMasterDbThread"))
    {
      CommandDispatcher *c = CommandDispatcher::getInstance();
      c->listenForCommand("save_to_cloud", this, mtSaveToCloud);
      c->listenForCommand("cloud_delete", this, mtCloudDelete);
      c->listenForCommand("cloud_revoke", this, mtCloudRevoke);

      start();
    }

  public:
    static NfsMasterDbThread *instance()
    {
      static NfsMasterDbThread *result = new NfsMasterDbThread();
      return result;
    }
  };

  class UserFiles
  {
  private:
    NewWorkerCluster &_workers;
    inline IContainerThread *getContainer() { return _workers.getContainer(); }

    std::string pathForUser(UserId userId) const
    {
      if (userId <= 0)
        return "";
      const int DIVISOR = 5;
      return "user_data/" + ntoa(DIVISOR) + '/' + ntoa(userId % DIVISOR)
	+ '/' + ntoa(userId);
    }

    std::string pathForUser(ExternalRequest const &m) const
    {
      SocketInfo *const socket = m.getSocketInfo();
      const UserId userId = userInfoGetInfo(socket).userId;
      return pathForUser(userId);
    }

    static std::unordered_set< std::string > const &legalFileNames()
    {
      static std::unordered_set< std::string > legal =
        {"custom_channels.xml", "channel_image_0.png", "channel_image_1.png",
         "channel_image_2.png", "channel_image_3.png", "channel_image_4.png"};
      return legal;
    }

    static bool isLegalFileName(std::string const &name)
    {
      return legalFileNames().count(name);
    }

    // Creates the path if it does not already exist.  Might create several
    // directories.
    // Returns true if the path appears to be in place,
    // appropriately permissioned, etc.  Otherwise returns false and writes
    // an error message to the log file.
    //
    // We are explicitly not going to sanitize the path.  The user gets some
    // input, but the path is generated by server code and we assume it is
    // safe.
    static bool ensurePathExists(std::string const &path)
    {
      const std::vector< std::string > elements = explode("/", path);
      std::string pathSoFar;
      for (std::string const &element : elements)
      {
        if (!pathSoFar.empty())
          pathSoFar += '/';
        pathSoFar += element;
        struct stat fileStatus;
        int statError = stat(pathSoFar.c_str(), &fileStatus);
        // For simplicity assume no error means that the directory already
        // exists and we should look at the next one and assume that an error
        // means that the directory is missing and we should create it now.
        // Both of those could be wrong.  But if we naïvely make these
        // assumptions and we're wrong, an explain...() call down the road will
        // explain the real problem.
        if (statError)
        {
          const mode_t MODE = 0755;  // rwxr-xr-x
          int mkdirError = mkdir(pathSoFar.c_str(), MODE);
          if (mkdirError)
          {
            *strerror = "System failure: unknown directory:  " + pathSoFar;
            std::cout<< *strerror <<std::endl;
            TclList msg;
            msg<<FLF<< *strerror;
            LogFile::primary().sendString(msg);
            return false;
          }
        }
      }
      assert(pathSoFar == path);
      return true;
    }

    void reportError(ExternalRequest const &m, std::string const &msg) const
    {
      XmlNode response;
      response["ERROR"].text = msg;
      addToOutputQueue(m.getSocketInfo(), response.asString("API"),
                       m.getResponseMessageId());
    }

    void userFileSave(ExternalRequest const &m) const
    {
      ThreadMonitor::SetState tm("userFileSave");
      const std::string fileName = m.getProperty("name");
      if (!isLegalFileName(fileName))
      {
        reportError(m, "Invalid file name");
        tm.increment("userFileSave() Invalid file name");
        return;
      }
      const std::string path = pathForUser(m);
      if (path.empty())
      {
        reportError(m, "Not logged in");
        tm.increment("userFileSave() Not logged in");
        return;
      }
      const std::string finalFileName = path + '/' + fileName;
      std::string const *const body = m.getPropertyPtr("body");
      if ((!body) || body->empty())
      { // Delete this file
        int unlinkError = unlink(finalFileName.c_str());
        if (!unlinkError)
          tm.increment("deleted!");
        else if (errno == ENOENT)
        { // A component in pathname does not exist or is a dangling symbolic
          // link, or pathname is empty.
          //
          // Assume this means that the file we wanted to delete was already
          // deleted, or never existed.  So that's not an error.  Like asking
          // MySQL to delete something.
          tm.increment("delete not required");
          unlinkError = false;
        }
        if (unlinkError)
        {
          *strerror = "System warning: unknown pathname:  " + finalFileName;
          std::cout<< *strerror <<std::endl;
          TclList msg;
          msg<<FLF<<*strerror;
          LogFile::primary().sendString(msg, m.getSocketInfo());
          reportError(m, "Internal error");
        }
        else
          sendNullResponse(m);
      }
      else
      { // Save this file
        if (!ensurePathExists(path))
        {
          reportError(m, "Internal error");
          return;
        }
        const std::string tempFileName = finalFileName + ".tmp."
          + ntoa(getRandom64());
        int handle = creat(tempFileName.c_str(), PERMISSIONS);
        if (handle <= 0)
        {
          *strerror = "System failure: unable to create file:  " + tempFileName;
          std::cout<< *strerror <<std::endl;

          TclList msg;
          msg<<FLF<< *strerror;
          LogFile::primary().sendString(msg, m.getSocketInfo());
          // It's tempting to close the client connection rather than trying
          // to report an error.  If we close the connection without responding
          // to this message, the client would automatically retry.
          reportError(m, "Internal error");
          return;
        }
        size_t written = write(handle, body->c_str(), body->size());
        if (written != body->size())
        {
          *strerror = "System failure: unable to write data in file:  " + tempFileName;
          std::cout<< *strerror <<std::endl;
          TclList msg;
          msg<<FLF<<"PROBLEM IN write()"<<tempFileName<<body->size()<<written;
          if (((ssize_t)written) < 0)
            msg<<*strerror;
          LogFile::primary().sendString(msg, m.getSocketInfo());
          reportError(m, "Internal error");
          return;
        }
        int closeError = close(handle);
        if (closeError)
        { // This is possible.  According to the man page, close might report
          // an error because of a previous call to write().  This is
          // especially common with NFS.
          TclList msg;
          msg<<FLF<<"CAN'T close() FILE"<<tempFileName<<body->size()
             <<errorString();
          LogFile::primary().sendString(msg, m.getSocketInfo());
          reportError(m, "Internal error");
          return;
        }
        int renameError = rename(tempFileName.c_str(), finalFileName.c_str());
        if (renameError)
        {
          *strerror = "System failure: unable to rename file:  " + tempFileName;
          std::cout<< *strerror <<std::endl;
          TclList msg;
          msg<<FLF<<"CAN'T rename() FILE"
             <<*strerror;
          LogFile::primary().sendString(msg, m.getSocketInfo());
          reportError(m, "Internal error");
          return;
        }
        // Success!
        sendNullResponse(m);
      }
    }

    void userFileLoad(ExternalRequest const &m) const
    {
      ThreadMonitor::SetState tm("userFileLoad");
      std::string result;  // Send this to the client.
      const std::string fileName = m.getProperty("name");
      if (!isLegalFileName(fileName))
        tm.increment("userFileLoad() ILLEGAL FILE NAME");
      else
      { // The requested file name is valid.
        const std::string path = pathForUser(m);
        if (path.empty())
          tm.increment("userFileLoad() Not logged in");
        else
        { // User is logged in.  We know the path where we save his files.
          const std::string finalFileName = path + '/' + fileName;
          int handle = open(finalFileName.c_str(), O_RDONLY);
          if (handle < 0)
          {
            if (errno == ENOENT)
              // File (or one of the path elements) does not exist.
              tm.increment("user_file_load ENOENT");
            else
            { // An unexpected error.
              *strerror = "System failure: unable to open file:  " + finalFileName;
              std::cout<< *strerror <<std::endl;
              TclList msg;
              msg<<FLF<< *strerror;
              LogFile::primary().sendString(msg, m.getSocketInfo());
            }
          }
          else
          { // open() succeeded!
            off_t length = lseek(handle, 0, SEEK_END);
            if (length == (off_t)-1)
            {
              TclList msg;
              msg<<FLF<<"lseek()"<<finalFileName<<errorString();
              LogFile::primary().sendString(msg, m.getSocketInfo());
            }
            else
            {
              result.resize(length);
              ssize_t bytesRead = pread(handle, &result[0], length, 0);
              if (bytesRead == -1)
              {
                TclList msg;
                msg<<FLF<<"pread()"<<finalFileName<<errorString();
                LogFile::primary().sendString(msg, m.getSocketInfo());
              }
              else
              {
                if (bytesRead != length)
                {
                  result.clear();
                  // If this is false we already wrote off the end of the array!
                  assert(bytesRead < length);
                  TclList msg;
                  msg<<FLF<<"pread()"<<finalFileName<<bytesRead<<length;
                  LogFile::primary().sendString(msg, m.getSocketInfo());
                  // Something strange.  Presumably caught in the middle of an
                  // update.  Close the client connection so the client will
		  // retry.
                  DeleteSocketThread::deleteSocket(m.getSocketInfo());
                }
                // else Success!!!
              }
            }
            close(handle);  // TODO check for errors and report them?
          }
        }
      }
      // Either result is the contents of the requested file, or result is 
      // blank to report an error.
      addToOutputQueue(m.getSocketInfo(), result, m.getResponseMessageId());
    }

    void userFileList(ExternalRequest const &m) const
    {
      ThreadMonitor::SetState tm("userFileList");
      const std::string path = pathForUser(m);
      if (path.empty())
      {
        reportError(m, "Not logged in");
        tm.increment("userFileSave() Not logged in");
        return;
      }
      struct ToReport
      {
        std::string fileName;
        struct stat fileStatus;
      };
      std::vector< ToReport > toReport;
      auto const &possible = legalFileNames();
      toReport.reserve(possible.size() + 1);
      struct stat fileStatus;
      int statCount = 1;
      int statError = stat(path.c_str(), &fileStatus);
      if (statError)
      { // Can't find root directory for user.
        if (errno == ENOENT)
          // Not found.  We've never created a directory for this user.  Not
          // an error in our system.
          tm.increment("userFileList . ENOENT");
        else
        {
          *strerror = "System failure: unable to create directory:  " + path;
          std::cout<< *strerror <<std::endl;

          TclList msg;
          msg<<FLF<< *strerror;
          LogFile::primary().sendString(msg, m.getSocketInfo());
        }
      }
      else
      { // Directory seems to be working.  Now lets look for known files.
        toReport.emplace_back((ToReport){".", fileStatus});
        for (std::string const &fileName : possible)
        {
          const std::string fullPath = path + '/' + fileName;
          statCount++;
          int statError = stat(fullPath.c_str(), &fileStatus);
          if (statError)
          {
            if (errno != ENOENT)
            {
              *strerror = "System failure: unable to create directory:  " + fullPath;
              std::cout<< *strerror <<std::endl;

              TclList msg;
              msg<<FLF<< *strerror;
              LogFile::primary().sendString(msg, m.getSocketInfo());
              // Our error handling is rough and inconsistent from a client
              // perspective.  It's possible that we got half way through
              // the list before running into a server error.  In that case
              // we return a partial list with no indication of a problem.
              //
              // We do a pretty good job of recording stuff on the server.
              // That's easier.  We just log lots of stuff and a human sorts it
              // all out later.
              //
              // There are references elsewhere to disconnecting the user.
              // That makes sense in many cases.  Especially if this server
              // is mostly right, but one feature is broken (i.e. I forgot to
              // add one symbolic link when I set the server up) users who try
              // to use that feature and only those users will get pushed to
              // another server.  I'm worried about some of the down sides,
              // but it's probably worth trying here.
              break;
            }
          }
          else
            toReport.emplace_back((ToReport){fileName, fileStatus});
        }
      }
      tm.increment("stat()", statCount);
      // Report as much or little data as we've gathered.
      XmlNode response;
      if (!toReport.empty())
      {
        XmlNode &dirXml = response["DIRECTORY"];
        for (ToReport const &current : toReport)
        {
          XmlNode &fileXml = dirXml[-1];
          fileXml.properties["NAME"] = current.fileName;
          if (S_ISREG(current.fileStatus.st_mode))
            // Only display the size for regular files, not directories.
            fileXml.properties["SIZE"] =
              ntoa(current.fileStatus.st_size); // in bytes
          fileXml.properties["MODIFIED"] =
            ntoa(current.fileStatus.st_mtim.tv_sec); // time_t
        }
      }
      addToOutputQueue(m.getSocketInfo(), response.asString("API"),
                       m.getResponseMessageId());
    }

    void listenForCommand(std::string const &name,
                          std::function< void(ExternalRequest const &) >const &
                          callback)
    {
      CommandDispatcher::getInstance()
        ->listenForCommand(name,
                           new ForeverThreadUser::Listener< ExternalRequest >
                           (getContainer(), callback), 0);
    }

    // This is an attempt to make listenForCommand() even easier to use.
    // Now you don't have to use the lambda syntax (which still seems crazy to
    // me) each time you call listenForCommand().  This one method takes care
    // of all of the lambdas.
    //
    // This still seems lacking, somehow.  :(  Maybe I could move this from
    // a method to a global template that could work on any class.  Then I'd 
    // have to explicitly include the this pointer.  Maybe that's not terrible.
    void listenForCommand(std::string const &name,
			  void (UserFiles::*method)
			  (ExternalRequest const &) const)
    {
      listenForCommand(name, [=](ExternalRequest const &m) 
		       { (this->*method)(m); });
    }			  

  public:
    UserFiles(NewWorkerCluster &workers) : _workers(workers)
    {
      listenForCommand("user_file_save", &UserFiles::userFileSave);
      listenForCommand("user_file_load", &UserFiles::userFileLoad);
      listenForCommand("user_file_list", &UserFiles::userFileList);
    }
  };

  static const std::string s_mtCloudList = "mtCloudList";
  static const std::string s_mtCloudList_null = "mtCloudList null";
  static const std::string s_mtCloudLoad = "mtCloudLoad";
  static const std::string s_mtCloudLoad_null = "mtCloudLoad null";
  static const std::string s_mtCloudImport = "mtCloudImport";

  static const std::string s_getLayoutBody = "getLayoutBody";
  static const std::string s_getLayoutBody_reading = "getLayoutBody reading";
  static const std::string s_getLayoutBody_decompressing =
    "getLayoutBody decompressing";
  static const std::string s_getLayoutBody_notCompressed =
    "getLayoutBody not compressed";
  static const std::string s_getLayoutBody_normal =
    "getLayoutBody normal";
  static const std::string s_getLayoutBody_entryNotFound =
    "getLayoutBody not found";
  static const std::string s_getLayoutBody_fileError =
    "getLayoutBody file ERROR";
  static const std::string s_getLayoutBody_error = "getLayoutBody ERROR";


  class NfsSlaveDbThread : public ForeverThreadUser
  {
  private:
    enum { mtCloudList, mtCloudLoad, mtCloudImport };

    typedef TSRefCount< ExternalRequest > ExternalRequestRef;

    NewWorkerCluster _workers;
    UserFiles _userFiles;

    static DatabaseWithRetry &database()
    {
      return *DatabaseForThread(DatabaseWithRetry::SLAVE);
    }

    static void doCloudList(ExternalRequestRef const &original)
    {
      ThreadMonitor::SetState tm(s_mtCloudList);
      tm.increment(s_mtCloudList);
      SocketInfo *const socket = original->getSocketInfo();
      const bool easternTime = userInfoUseEasternTime(socket);
      const UserId userId = userInfoGetInfo(socket).userId;
      std::string sql =
        "SELECT strategy_id, share_code, icon, clear_previous_layout, "
        "fill_screen, short_description, long_description, window_count, "
        "UNIX_TIMESTAMP(creation) as creation "
        "FROM cloud_layout "
        "WHERE user_id=" + ntoa(userId);
      XmlNode response;
      XmlNode &rows = response["ROWS"];
      for (MysqlResultRef result = database().tryQueryUntilSuccess(sql);
           result->rowIsValid();
           result->nextRow())
      {
        XmlNode &row = rows[-1];
        row.properties["ID"] = result->getStringField("strategy_id");
        row.properties["CODE"] =
          CLOUD_PREFIX + result->getStringField("share_code");
        row.properties["ICON"] = result->getStringField("icon");
        row.properties["CLEAR_PREVIOUS_LAYOUT"] =
          (result->getStringField("clear_previous_layout") == "Y")?"1":"0";
        row.properties["FILL_SCREEN"] =
          (result->getStringField("fill_screen") == "Y")?"1":"0";
        row.properties["SHORT_DESCRIPTION"] =
          result->getStringField("short_description");
        row.properties["LONG_DESCRIPTION"] =
          result->getStringField("long_description");
        row.properties["WINDOW_COUNT"] =
          result->getStringField("window_count");
        time_t creation = result->getIntegerField("creation", 0);
        if (creation)
          row.properties["CREATION"] = exportTime(creation, easternTime);
      }
      addToOutputQueue(socket,
                       response.asString("API"),
                       original->getResponseMessageId());
    }

    static std::string getLayoutBody(int64_t strategyId, int64_t originalSize)
    {
      ThreadMonitor::SetState tm(s_getLayoutBody_reading);
      const std::string fileName = pathForFile(strategyId);
      if (fileName.empty())
      { // Not found.  This seems unavoidable.  This is different from the data
        // errors below.  Those suggest something more troubling, possibly a
        // programming error.
        tm.increment(s_getLayoutBody_entryNotFound);
        return "";
      }
      std::string body;
      if (!loadFromFile(fileName, body))
      { // Some type of internal error.  The database says it should be there
        // but we can't find it in the file system.  I suppose it could happen
        // because of a race condition, where another thread or server deleted
        // the item between when we went to the database and the file system,
        // but that should be rare.  If we see this a lot, that suggests a
        // problem.
        TclList msg;
        msg<<FLF<<"loadFromFile()"<<fileName<<errorString();
        LogFile::primary().sendString(msg);
        tm.increment(s_getLayoutBody_fileError);
        return "";
      }
      if (originalSize <= 0)
      { // Not compressed.
        tm.increment(s_getLayoutBody_notCompressed);
        return body;
      }
      tm.setState(s_getLayoutBody_decompressing);
      std::string result(originalSize, '\x00');
      uint64_t size = originalSize;
      int zlibResult =
        uncompress(reinterpret_cast<unsigned char *>(&result[0]), &size,
                   reinterpret_cast<const unsigned char *>(&body[0]),
                   body.length());
      if ((zlibResult != Z_OK) || (size != (uint64_t)originalSize))
      { // Some sort of error.  An inconsistency between the database and the
        // file system.  Return our standard "" as an indication of an error.
        tm.increment(s_getLayoutBody_error);
        TclList msg;
        msg<<FLF<<s_getLayoutBody_error<<"zlibResult"<<zlibResult
           <<"size found"<<size<<"size expected"<<originalSize
           <<"fileName"<<fileName;
        sendToLogFile(msg);
        return "";
      }
      else
      {
        tm.increment(s_getLayoutBody_normal);
        return result;
      }
    }

    static void getFromCloud(ExternalRequestRef const &original,
                             std::string const &where)
    {
      std::string sql =
        "SELECT clear_previous_layout, original_size, short_description, "
        "strategy_id, share_code, fill_screen "
        "FROM cloud_layout "
        "WHERE " + where;
      MysqlResultRef result = database().tryQueryUntilSuccess(sql);
      XmlNode response;
      XmlNode &layout = response["LAYOUT"];
      layout.useCdata = true;
      layout.properties["CLEAR_PREVIOUS"] =
        (result->getStringField("clear_previous_layout") == "Y")?"1":"0";
      layout.properties["FILL_SCREEN"] =
        (result->getStringField("fill_screen") == "Y")?"1":"0";
      layout.properties["NAME"] = result->getStringField("short_description");
      layout.text = getLayoutBody(result->getIntegerField("strategy_id", 0),
                                  result->getIntegerField("original_size", 0));
      addToOutputQueue(original->getSocketInfo(),
                       response.asString("API"),
                       original->getResponseMessageId());
    }

    static void doCloudLoad(ExternalRequestRef const &original)
    {
      ThreadMonitor::SetState tm(s_mtCloudLoad);
      tm.increment(s_mtCloudLoad);
      SocketInfo *const socket = original->getSocketInfo();
      const UserId userId = userInfoGetInfo(socket).userId;
      const int id = strtolDefault(original->getProperty("id"), 0);
      getFromCloud(original,
                   "user_id=" + ntoa(userId) + " AND strategy_id=" + ntoa(id));
    }

    static void doCloudImport(ExternalRequestRef const &original)
    {
      ThreadMonitor::SetState tm(s_mtCloudImport);
      tm.increment(s_mtCloudImport);
      std::string shortCode = original->getProperty("code");
      if (shortCode.length() > 32)
        shortCode.erase(0, shortCode.length() - 32);
      getFromCloud(original,
                   "share_code='" + mysqlEscapeString(shortCode) + "'");
    }


    virtual void initializeInThread()
    { // This is the dispatcher thread.  If you need a database, use a worker
      // thread.
      assertNoDatabaseThisThread();

      const int count =
        strtolDefault(getConfigItem("misc_ro_nfs_threads"), 4);
      assert(count > 0);
      _workers.createWorkersLambda([]() {
          // These tasks are explicitly aimed at a read only database.
          // Related tasks for the master database are handled elsewhere.
          disableDatabaseThisThread(DatabaseWithRetry::MASTER);
        }, count);
    }

    virtual void handleRequestInThread(Request *current)
    {
      ThreadMonitor &tm = ThreadMonitor::find();

      // Normally the container thread would delete the request.  Instead we're
      // using a reference counted pointer to handle the deletion.
      ExternalRequestRef const request =
        dynamic_cast< ExternalRequest * >(current);
      keepOriginal();

      // Notice the implied assertion.  If current is not of type
      // ExternalRequest we'd catch it here.  We'd have to change our memory
      // management if that was not true.
      SocketInfo *const socket = request->getSocketInfo();

      switch(current->callbackId)
      {
      case mtCloudList:
        {
          if (userInfoGetInfo(socket).userId)
          {
            tm.increment(s_mtCloudList);
            _workers.addJob1([request]() { doCloudList(request); }, socket);
          }
          else
          {
            tm.increment(s_mtCloudList_null);
            sendNullResponse(*request);
          }
          break;
        }
      case mtCloudLoad:
        {
          if (userInfoGetInfo(socket).userId)
          {
            tm.increment(s_mtCloudLoad);
            _workers.addJob1([request]() { doCloudLoad(request); }, socket);
          }
          else
          {
            tm.increment(s_mtCloudLoad_null);
            sendNullResponse(*request);
          }
          break;
        }
      case mtCloudImport:
        {
          tm.increment(s_mtCloudImport);
          _workers.addJob1([request]() { doCloudImport(request); }, socket);
          break;
        }
      default:
        abort();
      }
    }

    NfsSlaveDbThread() :
      ForeverThreadUser(IContainerThread::create("NfsSlaveDbThread")),
      _workers(getContainer(), "NfsSlaveDbThread"),
      _userFiles(_workers)
    {
      CommandDispatcher *c = CommandDispatcher::getInstance();
      c->listenForCommand("cloud_list", this, mtCloudList);
      c->listenForCommand("cloud_load", this, mtCloudLoad);
      c->listenForCommand("cloud_import", this, mtCloudImport);

      start();
    }

  public:
    static NfsSlaveDbThread *instance()
    {
      static NfsSlaveDbThread *result = new NfsSlaveDbThread();
      return result;
    }
  };
}

void initMiscNFS()
{
  if (getConfigItem("skip_nfs") == "1")
    // This seems essential.  The NFS code has some very specific requirements.
    // If there's a problem, maybe a test setup, maybe the NFS server is down,
    // we might want a way to skip this stuff and let the rest of the program
    // run.  That is NOT the default.  It's hard to report a half working
    // server.  By killing the program in case of trouble, and by testing as
    // soon as we start, we have the best chance of finding problems ASAP.
    return;
  if (!MiscNFS::testPaths())
    LogFile::primary().scheduleShutdown();
  MiscNFS::NfsMasterDbThread::instance();
  MiscNFS::NfsSlaveDbThread::instance();
}