#include #include "../shared/DatabaseWithRetry.h" #include "../shared/TwoDLookup.h" #include "../shared/GlobalConfigFile.h" #include "../shared/MarketHours.h" std::string getOutputDate() { time_t initialTime = time(NULL); initialTime += strtolDefault(getConfigItem("time_offset"), 0); struct tm brokenDown; localtime_r(&initialTime, &brokenDown); if (secondOfTheDay(initialTime) > 12 * MARKET_HOURS_HOUR) // Round to the nearest midnight. This is how some other overnight // scripts work. If you run them in the afternoon, we assume we are // doing it for tomorrow. If you rum them in the morning, we assume // we are doing to for later today. brokenDown.tm_mday++; brokenDown.tm_sec = 0; brokenDown.tm_min = 0; brokenDown.tm_hour = 0; brokenDown.tm_isdst = -1; while (true) { // In addition to returning a value, mktime normalizes the broken down // time structure. time_t newTime = mktime(&brokenDown); assert(newTime > 0); if ((brokenDown.tm_wday > 0) && (brokenDown.tm_wday < 6)) break; // If we land on a weekend, go forward to the next monday. brokenDown.tm_mday++; } char buffer[50]; sprintf(buffer, "%.4d-%.2d-%.2d", (int)brokenDown.tm_year + 1900, (int)brokenDown.tm_mon + 1, (int)brokenDown.tm_mday); std::cout< "< sql; sql.push_back("SELECT @before:=DATE(MAX(start)) FROM stocktwits_minutes"); sql.push_back("SELECT @days:=COUNT(DISTINCT DATE(start)) " "FROM stocktwits_minutes " "WHERE start BETWEEN @before - INTERVAL " + ntoa(days) + " DAY AND @before " "AND DAYOFWEEK(start) BETWEEN 2 AND 6"); sql.push_back("SELECT symbol,SUM(count)/(0.0+@days) AS C " "FROM stocktwits_minutes " "WHERE start BETWEEN @before - INTERVAL " + ntoa(days) + " DAY " "AND @before AND DAYOFWEEK(start) BETWEEN 2 AND 6 " "GROUP BY symbol"); std::vector< MysqlResultRef > results = st.tryAllUntilSuccess(sql.begin(), sql.end()); std::cout<<"field: "<getStringField(0) <<", @days: "<getStringField(0) < updateSql; updateSql.push_back("BEGIN"); updateSql.push_back("UPDATE alerts_daily SET " + field + " = 0 " "WHERE date = '" + outputDate + "'"); for (MysqlResultRef &result = results[2]; result->rowIsValid(); result->nextRow()) { const std::string origSymbol = result->getStringField(0); std::string translatedSymbol = translation.get("ours", origSymbol); if (translatedSymbol == "") translatedSymbol = origSymbol; const std::string socialAverage = result->getStringField(1); updateSql.push_back("UPDATE alerts_daily " "SET " + field + " = '" + mysqlEscapeString(socialAverage) + "' " "WHERE d_symbol = '" + mysqlEscapeString(translatedSymbol) + "' " "AND date = '" + outputDate + "'"); if (!csvField.empty()) outputTable.add(csvField, translatedSymbol, socialAverage); } updateSql.push_back("COMMIT"); live.tryAllUntilSuccess(updateSql.begin(), updateSql.end()); } static void getPreviousDay(DatabaseWithRetry &live, DatabaseWithRetry &st) { std::vector< std::string > sql; // Ideally look at the last full day of data. However, we want the ability // to run late in the evening, not just early in the morning. So if you // run after 6pm you will look at today. I expect the "early overnight" // to run after 6pm. The "late overnight" should finish just before 4:20am // when we first start running alerts. We do the best we can with the // partial data available in the evening, hoping that it will be replaced // in the morning. But the early one should be good just in case. sql.push_back("SELECT @cutoff := DATE(MAX(start) + INTERVAL 6 HOUR) " "FROM stocktwits_minutes"); // Look at the last trading day before the cutoff. We definately want to // skip weekends. Skipping a holiday isn't signficantly harder. Note that // the holidays table automatically skips weekends. sql.push_back("SELECT @date := MAX(day) FROM holidays " "WHERE day < @cutoff AND FIND_IN_SET('US', closed) = 0"); sql.push_back("SELECT symbol,SUM(count) AS C " "FROM stocktwits_minutes " "WHERE start BETWEEN @date and @date + INTERVAL 1 DAY " "GROUP BY symbol"); std::vector< MysqlResultRef > results = st.tryAllUntilSuccess(sql.begin(), sql.end()); std::cout<<"cut off: "<getStringField(0) <<", previous day: "<getStringField(0) < updateSql; updateSql.push_back("BEGIN"); updateSql.push_back("UPDATE alerts_daily SET social_d = 0 " "WHERE date = '" + outputDate + "'"); for (MysqlResultRef &result = results[2]; result->rowIsValid(); result->nextRow()) { const std::string origSymbol = result->getStringField(0); std::string translatedSymbol = translation.get("ours", origSymbol); if (translatedSymbol == "") translatedSymbol = origSymbol; const std::string socialAverage = result->getStringField(1); updateSql.push_back("UPDATE alerts_daily " "SET social_d = '" + mysqlEscapeString(socialAverage) + "' " "WHERE d_symbol = '" + mysqlEscapeString(translatedSymbol) + "' " "AND date = '" + outputDate + "'"); } updateSql.push_back("COMMIT"); live.tryAllUntilSuccess(updateSql.begin(), updateSql.end()); } int main(int argc, char *argv[]) { addConfigItemsFromCommandLine(argv + 1); configItemsComplete(); outputDate = mysqlEscapeString(getOutputDate()); translation.loadFromCSV(getConfigItem("translation_table", "stocktwits_translation.csv")); DatabaseWithRetry live("@live", "live"); DatabaseWithRetry st("@st", "st"); // Historically this was the only option. It was originaly 21 days, 3 weeks, // because that seemed like a good number. We had to double that when // something failed and we had a 3 week gap in the data. This will probably // go away at some point. getNDays(live, st, "social_average", 42, "messages per day"); // One quarter. getNDays(live, st, "social_q", 91); // One month. More precisely, 20 week days or 4 weeks. It's nice to know // that when comparing weeks and months. I.e. if you wanted to know the // average for the last month not counting the last week. getNDays(live, st, "social_m", 28); // One week. Always 7 calendar days. 2 always get thrown away, and 5 always // get used. getNDays(live, st, "social_w", 7); // getPreviousDay() has different logic from getNDays(). In this case we // have to be more careful to avoid weekends. getPreviousDay(live, st); outputTable.writeToCSV(getConfigItem("output_table", "stocktwits_daily.csv")); }