{"slug":"llm-generated-alpha-factors","qid":"rank_persistence","label":"Training rank against holdout result: 240 trials cut into fifths","post_title":"Can an LLM Find Alpha Factors?","post_url":"/blog/llm-generated-alpha-factors#q-rank_persistence","columns":["training_group","avg_in_sample_sharpe","avg_out_of_sample_sharpe"],"rows":[{"training_group":"best fifth in training","avg_in_sample_sharpe":0.66,"avg_out_of_sample_sharpe":0.01},{"training_group":"second fifth","avg_in_sample_sharpe":0.26,"avg_out_of_sample_sharpe":0.02},{"training_group":"middle fifth","avg_in_sample_sharpe":-0.01,"avg_out_of_sample_sharpe":-0.08},{"training_group":"fourth fifth","avg_in_sample_sharpe":-0.24,"avg_out_of_sample_sharpe":-0.03},{"training_group":"worst fifth in training","avg_in_sample_sharpe":-0.63,"avg_out_of_sample_sharpe":0.13}],"shape":"ranking","sql":"WITH month_end AS (\n    SELECT ticker,\n           toStartOfMonth(toDate(toTimeZone(window_start, 'America/New_York'))) AS month_start,\n           argMax(toFloat64(close), window_start) AS close_px\n    FROM global_markets.delayed_stocks_minute_aggs\n    WHERE ticker IN ('AAPL','ADBE','AMZN','BA','CAT','COST','CRM','CSCO','CVX','DE',\n                     'DUK','GE','GOOGL','HD','HON','IBM','INTC','JNJ','JPM','KO',\n                     'LMT','MCD','MMM','MRK','MSFT','NKE','NVDA','ORCL','PEP','PFE',\n                     'PG','QCOM','SO','T','TGT','TXN','UNP','VZ','WMT','XOM')\n      AND toDate(toTimeZone(window_start, 'America/New_York')) >= toDate('2015-12-01')\n      AND toDate(toTimeZone(window_start, 'America/New_York')) <= toDate('2026-06-30')\n      AND toDayOfMonth(toTimeZone(window_start, 'America/New_York')) >= 22\n      AND (toHour(toTimeZone(window_start, 'America/New_York')) * 60\n           + toMinute(toTimeZone(window_start, 'America/New_York'))) BETWEEN 570 AND 959\n    GROUP BY ticker, month_start\n),\nlagged AS (\n    SELECT ticker,\n           month_start,\n           close_px,\n           lagInFrame(close_px) OVER (PARTITION BY ticker ORDER BY month_start\n                                      ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS prev_px\n    FROM month_end\n),\nmonthly_return AS (\n    SELECT ticker, month_start, close_px / prev_px - 1 AS ret\n    FROM lagged\n    WHERE prev_px > 0\n      AND month_start >= toDate('2016-01-01')\n),\ntrial AS (\n    SELECT arrayJoin(range(1, 241)) AS n\n),\nfactor_month AS (\n    SELECT t.n AS trial_id,\n           m.month_start AS month_start,\n           avgIf(m.ret, bitAnd(cityHash64(m.ticker, toString(m.month_start), t.n), 1) = 1)\n         - avgIf(m.ret, bitAnd(cityHash64(m.ticker, toString(m.month_start), t.n), 1) = 0) AS long_short_ret\n    FROM monthly_return AS m\n    CROSS JOIN trial AS t\n    GROUP BY trial_id, month_start\n    HAVING countIf(bitAnd(cityHash64(m.ticker, toString(m.month_start), t.n), 1) = 1) > 0\n       AND countIf(bitAnd(cityHash64(m.ticker, toString(m.month_start), t.n), 1) = 0) > 0\n),\nscored AS (\n    SELECT trial_id,\n           avgIf(long_short_ret, month_start <  toDate('2021-07-01'))\n             / stddevSampIf(long_short_ret, month_start <  toDate('2021-07-01')) * sqrt(12) AS in_sample_sharpe,\n           avgIf(long_short_ret, month_start >= toDate('2021-07-01'))\n             / stddevSampIf(long_short_ret, month_start >= toDate('2021-07-01')) * sqrt(12) AS out_of_sample_sharpe\n    FROM factor_month\n    GROUP BY trial_id\n    HAVING countIf(month_start <  toDate('2021-07-01')) >= 24\n       AND countIf(month_start >= toDate('2021-07-01')) >= 24\n),\nranked AS (\n    SELECT trial_id,\n           in_sample_sharpe,\n           out_of_sample_sharpe,\n           row_number() OVER (ORDER BY in_sample_sharpe DESC) AS in_sample_rank\n    FROM scored\n)\nSELECT multiIf(in_sample_rank <=  48, 'best fifth in training',\n               in_sample_rank <=  96, 'second fifth',\n               in_sample_rank <= 144, 'middle fifth',\n               in_sample_rank <= 192, 'fourth fifth',\n               'worst fifth in training') AS training_group,\n       round(avg(in_sample_sharpe), 2) AS avg_in_sample_sharpe,\n       round(avg(out_of_sample_sharpe), 2) AS avg_out_of_sample_sharpe\nFROM ranked\nGROUP BY training_group\nORDER BY min(in_sample_rank)","computed_at":"2026-08-02T08:36:39.937517+00:00","elapsed":53.65124696}