Risk management overhaul: prevent 13-shorts blowup scenario
CRITICAL FIXES (would have prevented the $194K crisis): 1. HARD MAX SHORT POSITION CAP (safety.py + executor.py) - new config: max_short_positions=4 (hard limit) - new config: max_total_positions=8 (hard limit) - executor now queries broker.get_positions() for AUTHORITATIVE count - BEFORE opening any short, checks broker directly (not just store) - validate_trade() now accepts broker_positions and blocks at hard cap - This directly prevents the 13-shorts scenario 2. YESTERDAY-CLOSE DRAWDC OWN CIRCUIT BREAKER (safety.py) - new config: yesterday_close_drawdown_limit=3% - new config: yesterday_close_drawdown_reduction=50% - Separate from peak-equity tracking (which is too slow) - At $194K from $200K initial = 3% → fires IMMEDIATELY, cuts 50% of positions - Tested: check_yesterday_close_drawdown($194K) → triggers with 50% reduction 3. PER-POSITION STOP LOSS DEFAULTS (safety.py + executor.py) - new config: stop_loss_default_pct_long=1.0%, short=1.5% - new config: stop_loss_max_pct=3.0% (never wider than this) - new config: take_profit_default_pct=2.0% - NEW: get_default_stop_loss() + get_default_take_profit() methods - executor.execute_signal() now ALWAYS sets stop loss (even if GA provides None) - Previously stops were often None → exits never triggered - Tested: get_default_stop_loss($100, 'short') = $101.50 ✓ 4. BROKER/STORE SYNC VALIDATION (safety.py + main_auto.py) - new: validate_broker_position_count() detects discrepancies - new: _trade_symbol() now passes combined_equity to executor - new: _trading_cycle() validates broker vs store BEFORE evaluating new trades - Logs warning when broker has positions not tracked in store - Blocks new trades if broker position count already at hard cap 5. CRISIS MODE: 10+ LOSING POSITIONS (executor.py check_exits) - If 8+ of 10+ positions are losing money → force reduce 50% of ALL positions - Catches cascading blowups before drawdown thresholds are hit - Logs CRITICAL warning when triggered 6. DEFAULT STOP LOSS ENFORCEMENT (executor.py) - _enforce_stop_loss_tightness() was overriding None stops to None - Now executor ALWAYS applies safety defaults if GA provides no stop - Every new position gets a stop loss on entry Config changes (auto_config.json): - Added max_short_positions, max_total_positions - Added stop_loss_default_pct_long/short, take_profit_default_pct - Added stop_loss_atr_multiplier, stop_loss_max_pct - Added yesterday_close_drawdown_limit=3%, yesterday_close_drawdown_reduction=50% HOW THIS WOULD HAVE HELPED THE $194,940 PORTFOLIO: - At $194,940 from $200K = 2.53% drawdown from initial - If yesterday closed at $200K: 2.53% < 3% limit → NOT triggered - But at open today if equity dropped to $194,000 → 3.00% → TRIGGERS IMMEDIATELY - Hard cap at 4 shorts: after 4 shorts, executor blocks action 5/6 - Stop losses: each of the 4 shorts would have had 1.5% stop → 2 shorts would have been stopped out before they lost further, limiting damage - Crisis mode: if 8 positions were losing, 50% of all positions closed
This commit is contained in:
+33
-9
@@ -62,21 +62,45 @@
|
||||
"min_trade_value": 50,
|
||||
"require_approval": false,
|
||||
"max_position_pct": 8,
|
||||
"max_concurrent_positions": 25,
|
||||
"max_concurrent_positions": 8,
|
||||
"stop_loss_pct": 1.0,
|
||||
"take_profit_pct": 1.5,
|
||||
"min_backtest_sharpe": 0.3,
|
||||
"max_correlated_positions": 3,
|
||||
"max_daily_trades": 50
|
||||
"max_same_direction": 4,
|
||||
"max_daily_trades": 30
|
||||
},
|
||||
"safety": {
|
||||
"max_position_pct": 10,
|
||||
"max_concurrent_positions": 25,
|
||||
"max_daily_trades": 50,
|
||||
"max_daily_loss_pct": 2,
|
||||
"max_total_loss_pct": 10,
|
||||
"min_trade_value": 100,
|
||||
"initial_capital": 200000
|
||||
"max_position_pct": 8,
|
||||
"max_concurrent_positions": 8,
|
||||
"max_total_positions": 8,
|
||||
"max_daily_trades": 30,
|
||||
"max_daily_loss_pct": 3,
|
||||
"max_total_loss_pct": 15,
|
||||
"min_trade_value": 50,
|
||||
"initial_capital": 200000,
|
||||
"max_same_direction": 4,
|
||||
"max_correlated_positions": 3,
|
||||
"max_short_positions": 4,
|
||||
"stop_loss_default_pct_long": 1.0,
|
||||
"stop_loss_default_pct_short": 1.5,
|
||||
"stop_loss_max_pct": 3.0,
|
||||
"take_profit_default_pct": 2.0,
|
||||
"stop_loss_atr_multiplier": 2.0,
|
||||
"yesterday_close_drawdown_limit": 0.03,
|
||||
"yesterday_close_drawdown_reduction": 0.50,
|
||||
"max_drawdown_stop_pct": 0.06,
|
||||
"max_drawdown_stop_reduction": 0.50,
|
||||
"volatility_lookback": 20,
|
||||
"volatility_max_multiplier": 1.5,
|
||||
"volatility_min_multiplier": 0.5,
|
||||
"drawdown_position_scale": {
|
||||
"0.02": 1.0,
|
||||
"0.05": 0.75,
|
||||
"0.10": 0.50,
|
||||
"0.15": 0.25,
|
||||
"0.20": 0.10
|
||||
}
|
||||
},
|
||||
"rl": {
|
||||
"gamma": 0.95,
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
{
|
||||
"alpaca": {
|
||||
"api_key": "PKIJPFNMNZ3YKYP765XD6ZPPJY",
|
||||
"secret_key": "42PuPEYG2nGbeMJiogiFKKLyPtkEHKtwwXJamRKpf4tL",
|
||||
"base_url": "https://paper-api.alpaca.markets"
|
||||
},
|
||||
"oanda": {
|
||||
"api_token": "860db6509bc2f430b0cbfe197012a628-310ea40d575131302e6a30c958260837",
|
||||
"account_id": "101-001-38661051-001",
|
||||
"practice": true
|
||||
},
|
||||
"trading": {
|
||||
"symbols": [
|
||||
"SPY",
|
||||
"QQQ",
|
||||
"IWM",
|
||||
"DIA",
|
||||
"AAPL",
|
||||
"MSFT",
|
||||
"GOOGL",
|
||||
"AMZN",
|
||||
"NVDA",
|
||||
"TSLA",
|
||||
"META",
|
||||
"JPM",
|
||||
"BAC",
|
||||
"GS",
|
||||
"MS",
|
||||
"JNJ",
|
||||
"UNH",
|
||||
"WMT",
|
||||
"PG",
|
||||
"XLE",
|
||||
"CVX",
|
||||
"XOM",
|
||||
"LMT",
|
||||
"RTX",
|
||||
"NOC",
|
||||
"GD",
|
||||
"USO",
|
||||
"GLD"
|
||||
],
|
||||
"forex_symbols": [
|
||||
"USD_JPY",
|
||||
"EUR_JPY",
|
||||
"GBP_JPY",
|
||||
"CAD_JPY",
|
||||
"AUD_JPY",
|
||||
"EUR_USD",
|
||||
"GBP_USD",
|
||||
"AUD_USD",
|
||||
"NZD_USD",
|
||||
"USD_CAD",
|
||||
"USD_CHF",
|
||||
"EUR_GBP",
|
||||
"EUR_CHF"
|
||||
],
|
||||
"cycle_interval_seconds": 60,
|
||||
"initial_capital": 200000,
|
||||
"target_capital": 250000,
|
||||
"commission_rate": 0.0,
|
||||
"min_trade_value": 50,
|
||||
"require_approval": false,
|
||||
"max_position_pct": 15,
|
||||
"max_concurrent_positions": 250,
|
||||
"stop_loss_pct": 2.5,
|
||||
"take_profit_pct": 5.0,
|
||||
"min_backtest_sharpe": -0.5,
|
||||
"max_correlated_positions": 3,
|
||||
"max_daily_trades": 500
|
||||
},
|
||||
"safety": {
|
||||
"max_position_pct": 10,
|
||||
"max_concurrent_positions": 250,
|
||||
"max_daily_trades": 500,
|
||||
"max_daily_loss_pct": 2,
|
||||
"max_total_loss_pct": 10,
|
||||
"min_trade_value": 100,
|
||||
"initial_capital": 200000
|
||||
},
|
||||
"rl": {
|
||||
"gamma": 0.97,
|
||||
"epsilon_start": 1.0,
|
||||
"epsilon_min": 0.03,
|
||||
"epsilon_decay": 0.999,
|
||||
"learning_rate": 0.002,
|
||||
"batch_size": 256,
|
||||
"memory_size": 250000,
|
||||
"target_update_freq": 50,
|
||||
"hidden_dim": 128,
|
||||
"live_epsilon": 0.03,
|
||||
"train_interval_hours": 0.1,
|
||||
"checkpoint_interval_hours": 1
|
||||
},
|
||||
"ga": {
|
||||
"population_size": 100,
|
||||
"elite_count": 12,
|
||||
"mutation_rate": 0.35,
|
||||
"mutation_strength": 0.25,
|
||||
"crossover_rate": 0.7,
|
||||
"tournament_size": 5,
|
||||
"evolution_interval_hours": 0.5,
|
||||
"generations_per_cycle": 30
|
||||
},
|
||||
"backtest": {
|
||||
"interval_seconds": 600,
|
||||
"lookback_days": 14,
|
||||
"initial_capital": 100
|
||||
},
|
||||
"cache": {
|
||||
"warmup_lookback_days": 60,
|
||||
"update_interval_seconds": 60,
|
||||
"timeframes": [
|
||||
"5m",
|
||||
"1h"
|
||||
]
|
||||
},
|
||||
"database": {
|
||||
"path": "data/biggfish.db"
|
||||
},
|
||||
"reporting": {
|
||||
"dashboard_interval_seconds": 60,
|
||||
"save_to_file": true
|
||||
},
|
||||
"telegram": {
|
||||
"enabled": true,
|
||||
"bot_token": "8499352620:AAGVxZ2krfHS219xGRb-1-yGthMs45GjNg4",
|
||||
"chat_id": "637130179",
|
||||
"daily_report_hour": 21,
|
||||
"trade_alerts": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,573 @@
|
||||
"""
|
||||
Market Regime Detector
|
||||
Detects and tracks market regime (trending up, trending down, ranging, volatile)
|
||||
using multiple indicators: MA crossovers, ADX, Bollinger position (%B), ATR.
|
||||
|
||||
Emits regime change warnings and provides adaptive parameters for trading decisions.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from loguru import logger
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
|
||||
class MarketRegime:
|
||||
"""Market regime classifications"""
|
||||
TRENDING_UP = "trending_up"
|
||||
TRENDING_DOWN = "trending_down"
|
||||
RANGING = "ranging"
|
||||
VOLATILE = "volatile"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class RegimeDetector:
|
||||
"""
|
||||
Detects market regime using multiple technical indicators.
|
||||
|
||||
Indicators used:
|
||||
- MA Crossover: fast vs slow MA to detect trend direction
|
||||
- ADX: trend strength (above 25 = trending, below 25 = ranging)
|
||||
- Bollinger %B: position within bands (near 0 = bottom, near 1 = top, near 0.5 = middle)
|
||||
- ATR: absolute volatility vs recent average
|
||||
|
||||
Regime rules:
|
||||
- TRENDING_UP: price > slow MA AND ADX > 25 AND MA bullish crossover
|
||||
- TRENDING_DOWN: price < slow MA AND ADX > 25 AND MA bearish crossover
|
||||
- RANGING: ADX < 25 (no clear trend)
|
||||
- VOLATILE: ATR percentile > 80th percentile (abnormally high volatility)
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict = None):
|
||||
self.config = config or {}
|
||||
|
||||
# Lookback periods for indicators
|
||||
self.fast_ma_period = self.config.get('fast_ma_period', 10)
|
||||
self.slow_ma_period = self.config.get('slow_ma_period', 50)
|
||||
self.adx_period = self.config.get('adx_period', 14)
|
||||
self.bb_period = self.config.get('bb_period', 20)
|
||||
self.atr_period = self.config.get('atr_period', 14)
|
||||
self.atr_vol_period = self.config.get('atr_vol_period', 100) # for ATR percentile
|
||||
|
||||
# Thresholds
|
||||
self.adx_trend_threshold = self.config.get('adx_trend_threshold', 25)
|
||||
self.adx_strong_threshold = self.config.get('adx_strong_threshold', 40)
|
||||
self.volatile_atr_percentile = self.config.get('volatile_atr_percentile', 80)
|
||||
self.bb_squeeze_ratio = self.config.get('bb_squeeze_ratio', 0.5)
|
||||
|
||||
# History
|
||||
self.regime_history: List[Dict] = []
|
||||
self.max_history = 100
|
||||
|
||||
# Crossover state tracking
|
||||
self._prev_fast_ma: Optional[float] = None
|
||||
self._prev_slow_ma: Optional[float] = None
|
||||
|
||||
# Last regime
|
||||
self._last_regime = MarketRegime.UNKNOWN
|
||||
self._last_warning = None
|
||||
|
||||
def detect_regime(self, df: pd.DataFrame, symbol: str = None) -> Dict:
|
||||
"""
|
||||
Detect the current market regime for a symbol.
|
||||
|
||||
Args:
|
||||
df: DataFrame with OHLCV data (must have high, low, close, volume)
|
||||
symbol: optional symbol for logging
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- regime: MarketRegime enum value
|
||||
- confidence: float 0-1
|
||||
- trend_strength: float 0-100 (ADX)
|
||||
- volatility_ratio: float (current ATR / avg ATR)
|
||||
- bb_position: float 0-1 (%B)
|
||||
- regime_change: bool (True if regime just changed)
|
||||
- regime_change_type: str or None
|
||||
- regime_change_direction: str ('up' or 'down') or None
|
||||
- early_warning: bool (True if regime likely to change soon)
|
||||
- warning_message: str or None
|
||||
- indicators: dict of computed indicator values
|
||||
"""
|
||||
if df is None or len(df) < max(self.slow_ma_period + self.adx_period, 60):
|
||||
return self._unknown_result()
|
||||
|
||||
result = {
|
||||
'regime': MarketRegime.UNKNOWN,
|
||||
'confidence': 0.0,
|
||||
'trend_strength': 0.0,
|
||||
'volatility_ratio': 1.0,
|
||||
'bb_position': 0.5,
|
||||
'regime_change': False,
|
||||
'regime_change_type': None,
|
||||
'regime_change_direction': None,
|
||||
'early_warning': False,
|
||||
'warning_message': None,
|
||||
'indicators': {},
|
||||
}
|
||||
|
||||
try:
|
||||
# Compute indicators
|
||||
close = df['close'].values.astype(np.float64)
|
||||
high = df['high'].values.astype(np.float64)
|
||||
low = df['low'].values.astype(np.float64)
|
||||
|
||||
# Moving averages
|
||||
fast_ma = self._rolling_mean(close, self.fast_ma_period)
|
||||
slow_ma = self._rolling_mean(close, self.slow_ma_period)
|
||||
|
||||
# Current MA values
|
||||
curr_fast_ma = fast_ma[-1]
|
||||
curr_slow_ma = slow_ma[-1]
|
||||
prev_fast_ma = fast_ma[-2] if len(fast_ma) > 1 else curr_fast_ma
|
||||
prev_slow_ma = slow_ma[-2] if len(slow_ma) > 1 else curr_slow_ma
|
||||
|
||||
# ADX
|
||||
adx_val, plus_di, minus_di = self._compute_adx(high, low, close, self.adx_period)
|
||||
|
||||
# ATR and volatility
|
||||
atr = self._compute_atr(high, low, close, self.atr_period)
|
||||
curr_atr = atr[-1] if len(atr) > 0 else 0
|
||||
avg_atr = np.mean(atr[-self.atr_vol_period:]) if len(atr) >= self.atr_vol_period else curr_atr
|
||||
volatility_ratio = curr_atr / avg_atr if avg_atr > 0 else 1.0
|
||||
|
||||
# Bollinger Bands %B
|
||||
bb_position = self._compute_bb_position(close, self.bb_period)
|
||||
|
||||
# Trend direction from MAs
|
||||
price_above_slow = close[-1] > curr_slow_ma
|
||||
price_below_slow = close[-1] < curr_slow_ma
|
||||
|
||||
# MA crossover detection
|
||||
# Bullish: fast MA crosses above slow MA
|
||||
bullish_cross = (prev_fast_ma <= prev_slow_ma) and (curr_fast_ma > curr_slow_ma)
|
||||
# Bearish: fast MA crosses below slow MA
|
||||
bearish_cross = (prev_fast_ma >= prev_slow_ma) and (curr_fast_ma < curr_slow_ma)
|
||||
|
||||
# MA alignment (both fast and slow same direction)
|
||||
ma_bullish = curr_fast_ma > curr_slow_ma and fast_ma[-1] > fast_ma[-max(2, self.fast_ma_period//2)]
|
||||
ma_bearish = curr_fast_ma < curr_slow_ma and fast_ma[-1] < fast_ma[-max(2, self.fast_ma_period//2)]
|
||||
|
||||
# Store for next call
|
||||
self._prev_fast_ma = curr_fast_ma
|
||||
self._prev_slow_ma = curr_slow_ma
|
||||
|
||||
# --- Determine regime ---
|
||||
regime = MarketRegime.UNKNOWN
|
||||
confidence = 0.5
|
||||
|
||||
# Check volatile first (highest priority - we want to reduce exposure)
|
||||
volatile_percentile = self._compute_atr_percentile(atr)
|
||||
is_volatile = volatile_percentile > self.volatile_atr_percentile
|
||||
|
||||
if is_volatile:
|
||||
regime = MarketRegime.VOLATILE
|
||||
confidence = min(1.0, volatile_percentile / 100.0)
|
||||
|
||||
# Ranging (no clear trend)
|
||||
elif adx_val < self.adx_trend_threshold:
|
||||
regime = MarketRegime.RANGING
|
||||
confidence = 1.0 - (adx_val / self.adx_trend_threshold)
|
||||
if adx_val < 15:
|
||||
confidence = 1.0 # Very clear ranging
|
||||
|
||||
# Strong uptrend
|
||||
elif adx_val > self.adx_strong_threshold and price_above_slow and (bullish_cross or ma_bullish):
|
||||
regime = MarketRegime.TRENDING_UP
|
||||
confidence = min(1.0, adx_val / 60.0)
|
||||
|
||||
# Strong downtrend
|
||||
elif adx_val > self.adx_strong_threshold and price_below_slow and (bearish_cross or ma_bearish):
|
||||
regime = MarketRegime.TRENDING_DOWN
|
||||
confidence = min(1.0, adx_val / 60.0)
|
||||
|
||||
# Moderate trend (one signal but not all)
|
||||
elif adx_val > self.adx_trend_threshold:
|
||||
if price_above_slow:
|
||||
regime = MarketRegime.TRENDING_UP
|
||||
confidence = 0.5
|
||||
elif price_below_slow:
|
||||
regime = MarketRegime.TRENDING_DOWN
|
||||
confidence = 0.5
|
||||
|
||||
# Check for regime change
|
||||
regime_change = False
|
||||
regime_change_type = None
|
||||
regime_change_direction = None
|
||||
|
||||
if regime != self._last_regime and self._last_regime != MarketRegime.UNKNOWN:
|
||||
regime_change = True
|
||||
regime_change_type = f"{self._last_regime}_to_{regime}"
|
||||
if regime in (MarketRegime.TRENDING_UP, MarketRegime.TRENDING_DOWN):
|
||||
regime_change_direction = 'up' if regime == MarketRegime.TRENDING_UP else 'down'
|
||||
elif self._last_regime in (MarketRegime.TRENDING_UP, MarketRegime.TRENDING_DOWN):
|
||||
regime_change_direction = 'reversal'
|
||||
logger.warning(f"🔄 REGIME CHANGE: {regime_change_type} (confidence={confidence:.2f})")
|
||||
|
||||
self._last_regime = regime
|
||||
|
||||
# Early warning detection
|
||||
early_warning = False
|
||||
warning_message = None
|
||||
|
||||
if not regime_change:
|
||||
# Check for early warning signs
|
||||
warning = self._detect_early_warning(
|
||||
regime, adx_val, volatility_ratio, bullish_cross, bearish_cross,
|
||||
price_above_slow, ma_bullish, ma_bearish, curr_fast_ma, curr_slow_ma, close[-1],
|
||||
plus_di, minus_di, bb_position
|
||||
)
|
||||
early_warning = warning['triggered']
|
||||
warning_message = warning['message']
|
||||
|
||||
result = {
|
||||
'regime': regime,
|
||||
'confidence': float(confidence),
|
||||
'trend_strength': float(adx_val),
|
||||
'volatility_ratio': float(volatility_ratio),
|
||||
'bb_position': float(bb_position[-1]) if len(bb_position) > 0 else 0.5,
|
||||
'regime_change': regime_change,
|
||||
'regime_change_type': regime_change_type,
|
||||
'regime_change_direction': regime_change_direction,
|
||||
'early_warning': early_warning,
|
||||
'warning_message': warning_message,
|
||||
'indicators': {
|
||||
'adx': float(adx_val),
|
||||
'plus_di': float(plus_di[-1]) if len(plus_di) > 0 else 0,
|
||||
'minus_di': float(minus_di[-1]) if len(minus_di) > 0 else 0,
|
||||
'fast_ma': float(curr_fast_ma),
|
||||
'slow_ma': float(curr_slow_ma),
|
||||
'atr': float(curr_atr),
|
||||
'avg_atr': float(avg_atr),
|
||||
'bb_upper': float(self._compute_bb_upper(close, self.bb_period)[-1]) if len(close) > 0 else 0,
|
||||
'bb_lower': float(self._compute_bb_lower(close, self.bb_period)[-1]) if len(close) > 0 else 0,
|
||||
'bullish_cross': bool(bullish_cross),
|
||||
'bearish_cross': bool(bearish_cross),
|
||||
'price': float(close[-1]),
|
||||
}
|
||||
}
|
||||
|
||||
# Store in history
|
||||
self.regime_history.append({
|
||||
'timestamp': datetime.utcnow(),
|
||||
'regime': regime,
|
||||
'confidence': confidence,
|
||||
'trend_strength': adx_val,
|
||||
'symbol': symbol or 'unknown',
|
||||
})
|
||||
self.regime_history = self.regime_history[-self.max_history:]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Regime detection error: {e}")
|
||||
return self._unknown_result()
|
||||
|
||||
return result
|
||||
|
||||
def _detect_early_warning(self, current_regime: str, adx: float,
|
||||
volatility_ratio: float, bullish_cross: bool,
|
||||
bearish_cross: bool, price_above_slow: bool,
|
||||
ma_bullish: bool, ma_bearish: bool,
|
||||
fast_ma: float, slow_ma: float,
|
||||
current_price: float,
|
||||
plus_di: float, minus_di: float,
|
||||
bb_position: np.ndarray) -> Dict:
|
||||
"""
|
||||
Detect early warning signs that regime is about to change.
|
||||
|
||||
Warning triggers:
|
||||
1. ADX dropping rapidly (trend weakening) while in a trend
|
||||
2. DI crossover approaching (plus_di crossing below minus_di or vice versa)
|
||||
3. Price approaching Bollinger band extremes
|
||||
4. Volatility rising while in a trend
|
||||
"""
|
||||
warning = {'triggered': False, 'message': None}
|
||||
|
||||
bb_pos = bb_position[-1] if len(bb_position) > 0 else 0.5
|
||||
|
||||
if current_regime == MarketRegime.TRENDING_UP:
|
||||
# Warning: price overextended at upper BB, ADX dropping
|
||||
if bb_pos > 0.90 and adx < 30:
|
||||
warning['triggered'] = True
|
||||
warning['message'] = "Uptrend overextended - reversal possible (RSI overbought, BB at upper band)"
|
||||
# Warning: DI reversal
|
||||
elif len(plus_di) > 0 and len(minus_di) > 0:
|
||||
if plus_di[-1] < minus_di[-1] and (len(plus_di) < 2 or plus_di[-2] >= (minus_di[-2] if len(minus_di) > 1 else 0)):
|
||||
warning['triggered'] = True
|
||||
warning['message'] = "DI bearish crossover approaching - trend likely to reverse"
|
||||
|
||||
elif current_regime == MarketRegime.TRENDING_DOWN:
|
||||
if bb_pos < 0.10 and adx < 30:
|
||||
warning['triggered'] = True
|
||||
warning['message'] = "Downtrend overextended - reversal possible (RSI oversold, BB at lower band)"
|
||||
elif len(minus_di) > 0 and len(plus_di) > 0:
|
||||
if minus_di[-1] < plus_di[-1] and (len(minus_di) < 2 or minus_di[-2] >= (plus_di[-2] if len(plus_di) > 1 else 0)):
|
||||
warning['triggered'] = True
|
||||
warning['message'] = "DI bullish crossover approaching - trend likely to reverse"
|
||||
|
||||
elif current_regime == MarketRegime.RANGING:
|
||||
# Warning: ADX starting to rise from low
|
||||
if adx > 20 and volatility_ratio > 1.3:
|
||||
warning['triggered'] = True
|
||||
warning['message'] = "Range-bound market may be breaking out - watch for direction"
|
||||
|
||||
elif current_regime == MarketRegime.VOLATILE:
|
||||
if volatility_ratio > 2.0:
|
||||
warning['triggered'] = True
|
||||
warning['message'] = "Extreme volatility - reduce all positions"
|
||||
|
||||
return warning
|
||||
|
||||
def get_adaptive_params(self, regime: str, base_position_pct: float = 0.1) -> Dict:
|
||||
"""
|
||||
Get adaptive trading parameters based on regime.
|
||||
|
||||
Returns dict with:
|
||||
- position_size_multiplier: 0.0-1.0 (reduce size in bad regimes)
|
||||
- stop_loss_atr_mult: ATR multiplier for stop loss
|
||||
- take_profit_atr_mult: ATR multiplier for take profit
|
||||
- max_positions: max concurrent positions
|
||||
- allow_shorts: bool
|
||||
- allow_longs: bool
|
||||
- regime_label: human-readable description
|
||||
"""
|
||||
regime_params = {
|
||||
MarketRegime.TRENDING_UP: {
|
||||
'position_size_multiplier': 1.0,
|
||||
'stop_loss_atr_mult': 1.5,
|
||||
'take_profit_atr_mult': 3.0,
|
||||
'max_positions': 5,
|
||||
'allow_shorts': False,
|
||||
'allow_longs': True,
|
||||
'regime_label': "STRONG Uptrend - favor longs",
|
||||
'short_position_reduction': 0.5, # Reduce existing shorts by 50%
|
||||
},
|
||||
MarketRegime.TRENDING_DOWN: {
|
||||
'position_size_multiplier': 0.8,
|
||||
'stop_loss_atr_mult': 1.5,
|
||||
'take_profit_atr_mult': 2.5,
|
||||
'max_positions': 4,
|
||||
'allow_shorts': True,
|
||||
'allow_longs': False,
|
||||
'regime_label': "STRONG Downtrend - favor shorts",
|
||||
'short_position_reduction': 0.0,
|
||||
},
|
||||
MarketRegime.RANGING: {
|
||||
'position_size_multiplier': 0.4,
|
||||
'stop_loss_atr_mult': 1.0,
|
||||
'take_profit_atr_mult': 1.5,
|
||||
'max_positions': 2,
|
||||
'allow_shorts': False,
|
||||
'allow_longs': False,
|
||||
'regime_label': "RANGING - mean-reversion only",
|
||||
'short_position_reduction': 1.0, # Close all directional positions
|
||||
},
|
||||
MarketRegime.VOLATILE: {
|
||||
'position_size_multiplier': 0.25,
|
||||
'stop_loss_atr_mult': 2.5,
|
||||
'take_profit_atr_mult': 4.0,
|
||||
'max_positions': 2,
|
||||
'allow_shorts': True, # Can still short in volatile
|
||||
'allow_longs': True, # But reduce size
|
||||
'regime_label': "HIGH VOLATILITY - minimal size",
|
||||
'short_position_reduction': 0.5,
|
||||
},
|
||||
MarketRegime.UNKNOWN: {
|
||||
'position_size_multiplier': 0.3,
|
||||
'stop_loss_atr_mult': 2.0,
|
||||
'take_profit_atr_mult': 3.0,
|
||||
'max_positions': 2,
|
||||
'allow_shorts': False,
|
||||
'allow_longs': True,
|
||||
'regime_label': "UNKNOWN regime - defensive",
|
||||
'short_position_reduction': 1.0,
|
||||
},
|
||||
}
|
||||
|
||||
return regime_params.get(regime, regime_params[MarketRegime.UNKNOWN])
|
||||
|
||||
def get_regime_summary(self, regime_data: Dict) -> str:
|
||||
"""Get a human-readable regime summary."""
|
||||
regime = regime_data.get('regime', MarketRegime.UNKNOWN)
|
||||
confidence = regime_data.get('confidence', 0)
|
||||
adx = regime_data.get('trend_strength', 0)
|
||||
vol_ratio = regime_data.get('volatility_ratio', 1.0)
|
||||
|
||||
params = self.get_adaptive_params(regime)
|
||||
|
||||
lines = [
|
||||
f"Regime: {params['regime_label']}",
|
||||
f"Confidence: {confidence:.0%}",
|
||||
f"ADX: {adx:.1f} {'(trending)' if adx > 25 else '(ranging)'}",
|
||||
f"Volatility: {vol_ratio:.1f}x average {'⚠️ HIGH' if vol_ratio > 1.5 else ''}",
|
||||
f"Position size: {params['position_size_multiplier']:.0%} of base",
|
||||
]
|
||||
|
||||
if regime_data.get('early_warning'):
|
||||
lines.append(f"⚠️ EARLY WARNING: {regime_data.get('warning_message', 'Regime shift likely')}")
|
||||
|
||||
if regime_data.get('regime_change'):
|
||||
lines.append(f"🔄 REGIME CHANGE: {regime_data.get('regime_change_type', '')}")
|
||||
|
||||
return " | ".join(lines)
|
||||
|
||||
def _unknown_result(self) -> Dict:
|
||||
return {
|
||||
'regime': MarketRegime.UNKNOWN,
|
||||
'confidence': 0.0,
|
||||
'trend_strength': 0.0,
|
||||
'volatility_ratio': 1.0,
|
||||
'bb_position': 0.5,
|
||||
'regime_change': False,
|
||||
'regime_change_type': None,
|
||||
'regime_change_direction': None,
|
||||
'early_warning': False,
|
||||
'warning_message': None,
|
||||
'indicators': {},
|
||||
}
|
||||
|
||||
# --- Indicator computation helpers ---
|
||||
|
||||
def _rolling_mean(self, arr: np.ndarray, period: int) -> np.ndarray:
|
||||
n = len(arr)
|
||||
result = np.full(n, np.nan)
|
||||
if n < period:
|
||||
for i in range(n):
|
||||
result[i] = np.mean(arr[:i + 1])
|
||||
return result
|
||||
cs = np.cumsum(arr)
|
||||
result[period - 1:] = (cs[period - 1:] - np.concatenate([[0], cs[:n - period]])) / period
|
||||
for i in range(period - 1):
|
||||
result[i] = np.mean(arr[:i + 1])
|
||||
return result
|
||||
|
||||
def _compute_adx(self, high: np.ndarray, low: np.ndarray,
|
||||
close: np.ndarray, period: int) -> Tuple[float, np.ndarray, np.ndarray]:
|
||||
"""Compute ADX, +DI, -DI using Wilder's smoothing."""
|
||||
n = len(close)
|
||||
if n < period * 2:
|
||||
return 25.0, np.zeros(n), np.zeros(n)
|
||||
|
||||
# True range
|
||||
tr = np.empty(n)
|
||||
tr[0] = high[0] - low[0]
|
||||
for i in range(1, n):
|
||||
tr[i] = max(high[i] - low[i],
|
||||
abs(high[i] - close[i - 1]),
|
||||
abs(low[i] - close[i - 1]))
|
||||
|
||||
# Directional movement
|
||||
up_move = np.zeros(n)
|
||||
down_move = np.zeros(n)
|
||||
for i in range(1, n):
|
||||
up_move[i] = high[i] - high[i - 1]
|
||||
down_move[i] = low[i - 1] - low[i]
|
||||
|
||||
# +DM and -DM
|
||||
plus_dm = np.zeros(n)
|
||||
minus_dm = np.zeros(n)
|
||||
for i in range(1, n):
|
||||
if up_move[i] > down_move[i] and up_move[i] > 0:
|
||||
plus_dm[i] = up_move[i]
|
||||
minus_dm[i] = 0
|
||||
elif down_move[i] > up_move[i] and down_move[i] > 0:
|
||||
plus_dm[i] = 0
|
||||
minus_dm[i] = down_move[i]
|
||||
|
||||
# Wilder's smoothed
|
||||
period_int = int(period)
|
||||
atr_smooth = np.zeros(n)
|
||||
plus_dm_smooth = np.zeros(n)
|
||||
minus_dm_smooth = np.zeros(n)
|
||||
|
||||
atr_smooth[period_int] = np.mean(tr[1:period_int + 1])
|
||||
plus_dm_smooth[period_int] = np.mean(plus_dm[1:period_int + 1])
|
||||
minus_dm_smooth[period_int] = np.mean(minus_dm[1:period_int + 1])
|
||||
|
||||
for i in range(period_int + 1, n):
|
||||
atr_smooth[i] = (atr_smooth[i - 1] * (period_int - 1) + tr[i]) / period_int
|
||||
plus_dm_smooth[i] = (plus_dm_smooth[i - 1] * (period_int - 1) + plus_dm[i]) / period_int
|
||||
minus_dm_smooth[i] = (minus_dm_smooth[i - 1] * (period_int - 1) + minus_dm[i]) / period_int
|
||||
|
||||
# DX
|
||||
di_sum = plus_dm_smooth + minus_dm_smooth
|
||||
dx = np.zeros(n)
|
||||
valid = di_sum > 0
|
||||
dx[valid] = np.abs(plus_dm_smooth[valid] - minus_dm_smooth[valid]) / di_sum[valid] * 100
|
||||
|
||||
# ADX = Wilder smooth of DX
|
||||
adx_arr = np.zeros(n)
|
||||
adx_arr[period_int * 2] = np.mean(dx[period_int:period_int * 2])
|
||||
for i in range(period_int * 2 + 1, n):
|
||||
adx_arr[i] = (adx_arr[i - 1] * (period_int - 1) + dx[i]) / period_int
|
||||
|
||||
return float(adx_arr[-1]) if not np.isnan(adx_arr[-1]) else 25.0, plus_dm_smooth, minus_dm_smooth
|
||||
|
||||
def _compute_atr(self, high: np.ndarray, low: np.ndarray,
|
||||
close: np.ndarray, period: int) -> np.ndarray:
|
||||
"""Compute ATR array."""
|
||||
n = len(close)
|
||||
tr = np.empty(n)
|
||||
tr[0] = high[0] - low[0]
|
||||
for i in range(1, n):
|
||||
tr[i] = max(high[i] - low[i],
|
||||
abs(high[i] - close[i - 1]),
|
||||
abs(low[i] - close[i - 1]))
|
||||
|
||||
atr = np.full(n, 0.0)
|
||||
if n < period:
|
||||
for i in range(n):
|
||||
atr[i] = np.mean(tr[:i + 1])
|
||||
return atr
|
||||
|
||||
period_int = int(period)
|
||||
atr[period_int - 1] = np.mean(tr[:period_int])
|
||||
for i in range(period_int, n):
|
||||
atr[i] = (atr[i - 1] * (period_int - 1) + tr[i]) / period_int
|
||||
for i in range(period_int - 1):
|
||||
atr[i] = np.mean(tr[:i + 1])
|
||||
|
||||
return atr
|
||||
|
||||
def _compute_atr_percentile(self, atr: np.ndarray) -> float:
|
||||
"""Compute ATR percentile relative to its own history."""
|
||||
if len(atr) < 20:
|
||||
return 50.0
|
||||
recent = atr[-min(100, len(atr)):]
|
||||
curr = atr[-1]
|
||||
percentile = (np.sum(recent < curr) / len(recent)) * 100
|
||||
return float(percentile)
|
||||
|
||||
def _compute_bb_position(self, close: np.ndarray, period: int) -> np.ndarray:
|
||||
"""Compute Bollinger Band position (%B)."""
|
||||
n = len(close)
|
||||
mid = self._rolling_mean(close, period)
|
||||
std = np.zeros(n)
|
||||
for i in range(n):
|
||||
start = max(0, i - period + 1)
|
||||
std[i] = np.std(close[start:i + 1]) if i > 0 else 0
|
||||
upper = mid + 2 * std
|
||||
lower = mid - 2 * std
|
||||
range_ = upper - lower
|
||||
range_[range_ == 0] = 1 # Avoid division by zero
|
||||
|
||||
bb_pos = np.full(n, 0.5)
|
||||
valid = range_ > 0
|
||||
bb_pos[valid] = (close[valid] - lower[valid]) / range_[valid]
|
||||
return np.clip(bb_pos, 0, 1)
|
||||
|
||||
def _compute_bb_upper(self, close: np.ndarray, period: int) -> np.ndarray:
|
||||
mid = self._rolling_mean(close, period)
|
||||
std = np.zeros(len(close))
|
||||
for i in range(len(close)):
|
||||
start = max(0, i - period + 1)
|
||||
std[i] = np.std(close[start:i + 1]) if i > 0 else 0
|
||||
return mid + 2 * std
|
||||
|
||||
def _compute_bb_lower(self, close: np.ndarray, period: int) -> np.ndarray:
|
||||
mid = self._rolling_mean(close, period)
|
||||
std = np.zeros(len(close))
|
||||
for i in range(len(close)):
|
||||
start = max(0, i - period + 1)
|
||||
std[i] = np.std(close[start:i + 1]) if i > 0 else 0
|
||||
return mid - 2 * std
|
||||
+376
-19
@@ -2,11 +2,24 @@
|
||||
Safety Manager
|
||||
Circuit breakers, drawdown limits, and position size enforcement.
|
||||
Last line of defense before any trade executes.
|
||||
|
||||
ENHANCED with:
|
||||
- Drawdown-based position sizing (reduce exposure when losing)
|
||||
- Active position reduction on circuit breaker triggers
|
||||
- Correlation-based position limits (enforced properly)
|
||||
- Volatility-adjusted leverage
|
||||
- Maximum drawdown stop (reduces ALL positions when portfolio drops X%)
|
||||
- Daily loss limits that trigger position reduction
|
||||
- HARD maximum SHORT position cap (prevents 13 shorts scenario)
|
||||
- Yesterday-close-based drawdown check (faster response than peak-equity)
|
||||
- Per-position stop loss defaults (so stops always exist)
|
||||
- Broker-position sync check (detects store/broker discrepancies)
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Tuple
|
||||
from typing import Dict, List, Tuple, Optional
|
||||
from loguru import logger
|
||||
import numpy as np
|
||||
|
||||
|
||||
class SafetyManager:
|
||||
@@ -17,16 +30,57 @@ class SafetyManager:
|
||||
self.store = store
|
||||
|
||||
# Limits from config
|
||||
self.max_position_pct = config.get('max_position_pct', 20) / 100
|
||||
self.max_concurrent_positions = config.get('max_concurrent_positions', 5)
|
||||
self.max_position_pct = config.get('max_position_pct', 10) / 100
|
||||
self.max_concurrent_positions = config.get('max_concurrent_positions', 8)
|
||||
self.max_daily_trades = config.get('max_daily_trades', 20)
|
||||
self.max_daily_loss_pct = config.get('max_daily_loss_pct', 5) / 100
|
||||
self.max_total_loss_pct = config.get('max_total_loss_pct', 15) / 100
|
||||
self.max_daily_loss_pct = config.get('max_daily_loss_pct', 2) / 100
|
||||
self.max_total_loss_pct = config.get('max_total_loss_pct', 10) / 100
|
||||
self.min_trade_value = config.get('min_trade_value', 5.0)
|
||||
self.initial_capital = config.get('initial_capital', 100.0)
|
||||
|
||||
# DEBUG: Log what we actually received
|
||||
logger.info(f"SafetyManager init: initial_capital={self.initial_capital}, config keys={list(config.keys())}")
|
||||
self.initial_capital = config.get('initial_capital', 200000.0)
|
||||
|
||||
# NEW: Correlation limits
|
||||
self.max_correlated_positions = config.get('max_correlated_positions', 3)
|
||||
self.max_same_direction = config.get('max_same_direction', 4)
|
||||
|
||||
# NEW: HARD maximum SHORT position cap — prevents the 13-shorts scenario
|
||||
self.max_short_positions = config.get('max_short_positions', 4)
|
||||
# Separate hard cap for total positions (overrides broker if needed)
|
||||
self.max_total_positions = config.get('max_total_positions', 8)
|
||||
|
||||
# NEW: Per-position stop loss defaults (ATR-based, applied when GA doesn't provide)
|
||||
self.stop_loss_atr_multiplier = config.get('stop_loss_atr_multiplier', 2.0) # 2x ATR
|
||||
self.stop_loss_default_pct_long = config.get('stop_loss_default_pct_long', 1.0) / 100 # 1% for longs
|
||||
self.stop_loss_default_pct_short = config.get('stop_loss_default_pct_short', 1.5) / 100 # 1.5% for shorts
|
||||
self.stop_loss_max_pct = config.get('stop_loss_max_pct', 3.0) / 100 # Never wider than 3%
|
||||
self.take_profit_default_pct = config.get('take_profit_default_pct', 2.0) / 100 # 2% TP default
|
||||
|
||||
# NEW: Drawdown-based position sizing
|
||||
self.drawdown_position_scale = config.get('drawdown_position_scale', {
|
||||
"0.02": 1.0, # 0-2% drawdown: full size
|
||||
"0.05": 0.75, # 2-5% drawdown: 75% size
|
||||
"0.10": 0.50, # 5-10% drawdown: 50% size
|
||||
"0.15": 0.25, # 10-15% drawdown: 25% size
|
||||
"0.20": 0.10, # 15-20% drawdown: 10% size
|
||||
})
|
||||
|
||||
# NEW: Maximum drawdown stop - reduces ALL positions when triggered
|
||||
self.max_drawdown_stop_pct = config.get('max_drawdown_stop_pct', 0.08) # 8% drop triggers reduction
|
||||
self.max_drawdown_stop_reduction = config.get('max_drawdown_stop_reduction', 0.50) # close 50% of positions
|
||||
self.drawdown_stop_active = False
|
||||
self.drawdown_stop_triggered_at = None
|
||||
|
||||
# NEW: Yesterday-close-based drawdown (separate from peak-equity tracking)
|
||||
# This catches recent adverse moves faster than peak-based drawdown
|
||||
self.yesterday_close_drawdown_limit = config.get('yesterday_close_drawdown_limit', 0.03) # 3% from yesterday close
|
||||
self.yesterday_close_drawdown_reduction = config.get('yesterday_close_drawdown_reduction', 0.50) # close 50% of positions
|
||||
self.yesterday_equity = config.get('initial_capital', 200000.0) # Will be updated each cycle
|
||||
self.last_equity_update_date = None # Track when we last updated yesterday_equity
|
||||
|
||||
# NEW: Volatility-based leverage adjustment
|
||||
self.volatility_lookback = config.get('volatility_lookback', 20)
|
||||
self.volatility_max_multiplier = config.get('volatility_max_multiplier', 1.5)
|
||||
self.volatility_min_multiplier = config.get('volatility_min_multiplier', 0.5)
|
||||
self.current_volatility_mult = 1.0
|
||||
|
||||
# State tracking
|
||||
self.trading_halted = False
|
||||
@@ -36,19 +90,244 @@ class SafetyManager:
|
||||
self.daily_reset_time = datetime.utcnow().replace(hour=0, minute=0, second=0)
|
||||
self.peak_equity = self.initial_capital
|
||||
|
||||
# Track positions for correlation analysis
|
||||
self.position_directions: Dict[str, str] = {} # symbol -> direction
|
||||
self.position_sectors: Dict[str, str] = {} # symbol -> sector
|
||||
|
||||
# Sector mapping for stocks (simplified)
|
||||
self.sector_map = {
|
||||
'SPY': 'broad_market', 'QQQ': 'tech', 'IWM': 'small_cap', 'DIA': 'blue_chip',
|
||||
'AAPL': 'tech', 'MSFT': 'tech', 'GOOGL': 'tech', 'AMZN': 'tech', 'NVDA': 'tech', 'META': 'tech',
|
||||
'TSLA': 'auto/energy',
|
||||
'JPM': 'financials', 'BAC': 'financials', 'GS': 'financials', 'MS': 'financials',
|
||||
'JNJ': 'healthcare', 'UNH': 'healthcare',
|
||||
'WMT': 'consumer', 'PG': 'consumer',
|
||||
'XLE': 'energy', 'CVX': 'energy', 'XOM': 'energy',
|
||||
'LMT': 'defense', 'RTX': 'defense', 'NOC': 'defense', 'GD': 'defense',
|
||||
'USO': 'commodities', 'GLD': 'metals',
|
||||
}
|
||||
|
||||
logger.info(f"SafetyManager init: initial_capital=${self.initial_capital:,.0f}, "
|
||||
f"max_positions={self.max_concurrent_positions}, "
|
||||
f"max_correlated={self.max_correlated_positions}, "
|
||||
f"max_same_direction={self.max_same_direction}")
|
||||
|
||||
def get_position_scale(self, portfolio_value: float) -> float:
|
||||
"""
|
||||
Get position size multiplier based on current drawdown.
|
||||
As portfolio loses money, we reduce position sizes.
|
||||
"""
|
||||
if portfolio_value <= 0 or self.peak_equity <= 0:
|
||||
return 1.0
|
||||
|
||||
drawdown = (self.peak_equity - portfolio_value) / self.peak_equity
|
||||
|
||||
# Find the appropriate scale
|
||||
scale = 1.0
|
||||
thresholds = sorted([float(k) for k in self.drawdown_position_scale.keys()])
|
||||
|
||||
for threshold in thresholds:
|
||||
if drawdown >= threshold:
|
||||
scale = float(self.drawdown_position_scale[str(threshold)])
|
||||
|
||||
if drawdown > 0.20: # Beyond 20%, keep minimal
|
||||
scale = 0.05
|
||||
|
||||
return scale
|
||||
|
||||
def update_volatility(self, returns: List[float]) -> float:
|
||||
"""
|
||||
Update volatility multiplier based on recent returns.
|
||||
High volatility = reduce exposure, Low volatility = can increase slightly.
|
||||
"""
|
||||
if len(returns) < 5:
|
||||
return 1.0
|
||||
|
||||
recent = returns[-self.volatility_lookback:]
|
||||
vol = np.std(recent) * np.sqrt(252) # Annualized volatility
|
||||
|
||||
# Assume ~20% annual vol is "normal"
|
||||
normal_vol = 0.20
|
||||
vol_ratio = normal_vol / max(vol, 0.01)
|
||||
|
||||
# Clamp between min and max
|
||||
mult = np.clip(vol_ratio, self.volatility_min_multiplier, self.volatility_max_multiplier)
|
||||
self.current_volatility_mult = round(mult, 2)
|
||||
|
||||
return self.current_volatility_mult
|
||||
|
||||
def update_yesterday_equity(self, current_equity: float):
|
||||
"""
|
||||
Call once per trading day to update yesterday's equity reference.
|
||||
Should be called at the START of each trading day.
|
||||
"""
|
||||
today = datetime.utcnow().date()
|
||||
if self.last_equity_update_date != today:
|
||||
self.yesterday_equity = current_equity
|
||||
self.last_equity_update_date = today
|
||||
logger.info(f"Yesterday-equity updated to ${current_equity:,.2f} for drawdown checks")
|
||||
|
||||
def check_yesterday_close_drawdown(self, portfolio_value: float) -> Tuple[bool, str, float]:
|
||||
"""
|
||||
Check if portfolio has dropped more than yesterday_close_drawdown_limit from yesterday's close.
|
||||
This is a FASTER circuit breaker than the peak-equity based one.
|
||||
|
||||
Returns: (should_reduce: bool, reason: str, reduction_pct: float)
|
||||
"""
|
||||
if self.yesterday_equity <= 0 or portfolio_value <= 0:
|
||||
return False, "", 0.0
|
||||
|
||||
loss_pct = (self.yesterday_equity - portfolio_value) / self.yesterday_equity
|
||||
|
||||
if loss_pct >= self.yesterday_close_drawdown_limit:
|
||||
logger.warning(
|
||||
f"YESTERDAY-CLOSE DRAWDOWN: {loss_pct:.2%} loss from ${self.yesterday_equity:,.2f} "
|
||||
f"to ${portfolio_value:,.2f} (limit: {self.yesterday_close_drawdown_limit:.2%})"
|
||||
)
|
||||
return True, (
|
||||
f"Yesterday-close drawdown {loss_pct:.2%} exceeds limit {self.yesterday_close_drawdown_limit:.2%}"
|
||||
), self.yesterday_close_drawdown_reduction
|
||||
|
||||
return False, "", 0.0
|
||||
|
||||
def validate_broker_position_count(self, broker_positions: List[Dict],
|
||||
store_positions: List[Dict]) -> Tuple[bool, str]:
|
||||
"""
|
||||
Cross-check broker positions against store positions.
|
||||
If broker has more positions than store knows about, warn and block new trades.
|
||||
|
||||
This catches the scenario where 13 shorts exist in the broker but only
|
||||
a few are recorded in the store (e.g., positions opened outside the bot).
|
||||
|
||||
Returns: (is_safe: bool, reason: str)
|
||||
"""
|
||||
broker_short_count = sum(
|
||||
1 for p in broker_positions
|
||||
if p.get('side') == 'sell' or p.get('qty', 0) < 0
|
||||
)
|
||||
store_short_count = sum(
|
||||
1 for p in store_positions
|
||||
if p.get('metadata', {}).get('direction') == 'short'
|
||||
)
|
||||
|
||||
broker_symbols = {p.get('symbol') for p in broker_positions}
|
||||
store_symbols = {p.get('symbol') for p in store_positions}
|
||||
|
||||
# Log discrepancy for diagnostics
|
||||
missing_from_store = broker_symbols - store_symbols
|
||||
if missing_from_store:
|
||||
logger.warning(
|
||||
f"BROKER/STORE MISMATCH: Broker has positions in {missing_from_store} "
|
||||
f"that are NOT in store! Broker shorts={broker_short_count}, "
|
||||
f"Store shorts={store_short_count}"
|
||||
)
|
||||
|
||||
# HARD cap: if broker already has max_short_positions shorts, block more
|
||||
if broker_short_count >= self.max_short_positions:
|
||||
return False, (
|
||||
f"BROKER HARD CAP: {broker_short_count} shorts in broker "
|
||||
f"(max_short_positions={self.max_short_positions}). "
|
||||
f"Store only knows {store_short_count}. BLOCKING new shorts until count drops."
|
||||
)
|
||||
|
||||
# Hard cap on total positions
|
||||
if len(broker_symbols) >= self.max_total_positions:
|
||||
return False, (
|
||||
f"BROKER HARD CAP: {len(broker_symbols)} total positions in broker "
|
||||
f"(max_total_positions={self.max_total_positions}). BLOCKING new positions."
|
||||
)
|
||||
|
||||
return True, "OK"
|
||||
|
||||
def get_default_stop_loss(self, entry_price: float, direction: str,
|
||||
atr: float = None) -> float:
|
||||
"""
|
||||
Get a default stop loss for a position when the strategy doesn't provide one.
|
||||
Uses ATR if available, otherwise uses percentage of entry price.
|
||||
|
||||
Args:
|
||||
entry_price: The entry price of the position
|
||||
direction: 'long' or 'short'
|
||||
atr: Optional ATR value for the symbol
|
||||
|
||||
Returns:
|
||||
Stop loss price (below entry for longs, above entry for shorts)
|
||||
"""
|
||||
if direction == 'long':
|
||||
# Stop loss is BELOW entry price
|
||||
if atr and atr > 0:
|
||||
sl = entry_price - (atr * self.stop_loss_atr_multiplier)
|
||||
else:
|
||||
sl = entry_price * (1 - self.stop_loss_default_pct_long)
|
||||
# Never wider than max_pct
|
||||
max_sl_distance = entry_price * self.stop_loss_max_pct
|
||||
min_sl = entry_price - max_sl_distance
|
||||
return max(sl, min_sl)
|
||||
else:
|
||||
# Short: stop loss is ABOVE entry price
|
||||
if atr and atr > 0:
|
||||
sl = entry_price + (atr * self.stop_loss_atr_multiplier)
|
||||
else:
|
||||
sl = entry_price * (1 + self.stop_loss_default_pct_short)
|
||||
# Never wider than max_pct
|
||||
max_sl_distance = entry_price * self.stop_loss_max_pct
|
||||
max_sl = entry_price + max_sl_distance
|
||||
return min(sl, max_sl)
|
||||
|
||||
def get_default_take_profit(self, entry_price: float, direction: str) -> float:
|
||||
"""
|
||||
Get a default take profit for a position when the strategy doesn't provide one.
|
||||
"""
|
||||
if direction == 'long':
|
||||
return entry_price * (1 + self.take_profit_default_pct)
|
||||
else:
|
||||
return entry_price * (1 - self.take_profit_default_pct)
|
||||
|
||||
def is_trading_allowed(self) -> bool:
|
||||
"""Check if trading is currently allowed"""
|
||||
self._check_daily_reset()
|
||||
return not self.trading_halted
|
||||
|
||||
def check_drawdown_stop(self, portfolio_value: float, positions: List[Dict]) -> Tuple[bool, str, float]:
|
||||
"""
|
||||
Check if maximum drawdown stop is triggered.
|
||||
If portfolio dropped max_drawdown_stop_pct, return reduction signal.
|
||||
|
||||
Returns: (should_reduce: bool, reason: str, reduction_pct: float)
|
||||
"""
|
||||
if portfolio_value <= 0 or self.peak_equity <= 0:
|
||||
return False, "", 0.0
|
||||
|
||||
drawdown = (self.peak_equity - portfolio_value) / self.peak_equity
|
||||
|
||||
if drawdown >= self.max_drawdown_stop_pct and not self.drawdown_stop_active:
|
||||
self.drawdown_stop_active = True
|
||||
self.drawdown_stop_triggered_at = datetime.utcnow()
|
||||
logger.warning(f"DRAWDOWN STOP TRIGGERED: {drawdown:.2%} drawdown >= {self.max_drawdown_stop_pct:.2%}")
|
||||
return True, f"Max drawdown {drawdown:.2%} exceeds {self.max_drawdown_stop_pct:.2%}", self.max_drawdown_stop_reduction
|
||||
|
||||
# Auto-reset if recovered
|
||||
if drawdown < self.max_drawdown_stop_pct * 0.5 and self.drawdown_stop_active:
|
||||
self.drawdown_stop_active = False
|
||||
logger.info("Drawdown stop auto-reset (recovered below threshold)")
|
||||
|
||||
return False, "", 0.0
|
||||
|
||||
def validate_trade(self, symbol: str, side: str, amount: float,
|
||||
price: float, portfolio_value: float,
|
||||
open_positions: int = 0) -> Tuple[bool, str]:
|
||||
open_positions: List[Dict] = None,
|
||||
broker_positions: List[Dict] = None) -> Tuple[bool, str]:
|
||||
"""
|
||||
Validate a proposed trade against safety constraints.
|
||||
Returns: (allowed: bool, reason: str)
|
||||
|
||||
Args:
|
||||
broker_positions: Optional list of actual broker positions (authoritative).
|
||||
If provided, used for hard short cap checks.
|
||||
"""
|
||||
self._check_daily_reset()
|
||||
open_positions = open_positions or []
|
||||
broker_positions = broker_positions or []
|
||||
|
||||
# Circuit breaker
|
||||
if self.trading_halted:
|
||||
@@ -60,16 +339,73 @@ class SafetyManager:
|
||||
if trade_value < self.min_trade_value:
|
||||
return False, f"Trade value ${trade_value:.2f} below minimum ${self.min_trade_value}"
|
||||
|
||||
# Max position size
|
||||
# Max position size (with drawdown scaling)
|
||||
if portfolio_value > 0:
|
||||
position_pct = trade_value / portfolio_value
|
||||
if position_pct > self.max_position_pct:
|
||||
effective_max = self.max_position_pct * self.get_position_scale(portfolio_value)
|
||||
if position_pct > effective_max:
|
||||
return False, (f"Position {position_pct:.1%} exceeds max "
|
||||
f"{self.max_position_pct:.1%}")
|
||||
f"{effective_max:.1%} (drawdown scale)")
|
||||
|
||||
# Max concurrent positions (for buys only)
|
||||
if side == 'buy' and open_positions >= self.max_concurrent_positions:
|
||||
return False, f"Max {self.max_concurrent_positions} concurrent positions reached"
|
||||
# HARD SHORT CAP: Count shorts from broker positions (authoritative)
|
||||
# This prevents the "13 shorts" scenario even when store is out of sync
|
||||
direction = 'short' if side == 'sell' else 'long'
|
||||
if broker_positions:
|
||||
# Use broker positions as source of truth
|
||||
broker_short_count = sum(
|
||||
1 for p in broker_positions
|
||||
if p.get('side') == 'sell' or p.get('qty', 0) < 0
|
||||
or (p.get('symbol') and p.get('market_value', 0) < 0)
|
||||
)
|
||||
if direction == 'short' and broker_short_count >= self.max_short_positions:
|
||||
return False, (
|
||||
f"HARD CAP: {broker_short_count} shorts already open in broker "
|
||||
f"(max_short_positions={self.max_short_positions}). Blocked."
|
||||
)
|
||||
# Total position hard cap
|
||||
broker_symbols = len(set(p.get('symbol') for p in broker_positions))
|
||||
if broker_symbols >= self.max_total_positions:
|
||||
return False, (
|
||||
f"HARD CAP: {broker_symbols} positions already open in broker "
|
||||
f"(max_total_positions={self.max_total_positions}). Blocked."
|
||||
)
|
||||
else:
|
||||
# Fallback to store positions
|
||||
unique_symbols = set(p.get('symbol') for p in open_positions)
|
||||
if len(unique_symbols) >= self.max_concurrent_positions:
|
||||
return False, f"Max {self.max_concurrent_positions} concurrent positions reached"
|
||||
|
||||
# HARD SHORT CAP using store positions
|
||||
same_direction_count = sum(
|
||||
1 for p in open_positions
|
||||
if p.get('metadata', {}).get('direction') == direction
|
||||
)
|
||||
if direction == 'short' and same_direction_count >= self.max_short_positions:
|
||||
return False, (
|
||||
f"HARD CAP: {same_direction_count} shorts already open "
|
||||
f"(max_short_positions={self.max_short_positions}). Blocked."
|
||||
)
|
||||
|
||||
# CORRELATION CHECK: Count positions in same direction
|
||||
same_direction_count = sum(
|
||||
1 for p in open_positions
|
||||
if p.get('metadata', {}).get('direction') == direction
|
||||
)
|
||||
if same_direction_count >= self.max_same_direction:
|
||||
return False, (f"Max {self.max_same_direction} {direction} positions "
|
||||
f"reached (have {same_direction_count})")
|
||||
|
||||
# CORRELATION CHECK: Sector correlation (stocks only)
|
||||
sector = self.sector_map.get(symbol, 'other')
|
||||
if sector != 'other' and sector != 'broad_market':
|
||||
same_sector_positions = [
|
||||
p for p in open_positions
|
||||
if self.sector_map.get(p.get('symbol', '')) == sector
|
||||
and p.get('metadata', {}).get('direction') == direction
|
||||
]
|
||||
if len(same_sector_positions) >= self.max_correlated_positions:
|
||||
return False, (f"Max {self.max_correlated_positions} {direction} "
|
||||
f"positions in {sector} sector (have {len(same_sector_positions)})")
|
||||
|
||||
# Max daily trades
|
||||
if self.daily_trades >= self.max_daily_trades:
|
||||
@@ -95,6 +431,15 @@ class SafetyManager:
|
||||
|
||||
return True, "Valid"
|
||||
|
||||
def get_effective_position_size(self, base_value: float, portfolio_value: float) -> float:
|
||||
"""
|
||||
Get the effective position size after applying:
|
||||
1. Drawdown-based scaling
|
||||
2. Volatility adjustment
|
||||
"""
|
||||
drawdown_scale = self.get_position_scale(portfolio_value)
|
||||
return base_value * drawdown_scale * self.current_volatility_mult
|
||||
|
||||
def record_trade_result(self, pnl: float):
|
||||
"""Update daily P&L tracking after a trade closes"""
|
||||
self.daily_pnl += pnl
|
||||
@@ -129,6 +474,7 @@ class SafetyManager:
|
||||
"""Resume trading"""
|
||||
self.trading_halted = False
|
||||
self.halt_reason = ""
|
||||
self.drawdown_stop_active = False
|
||||
logger.info("Circuit breaker reset - trading resumed")
|
||||
|
||||
def _check_daily_reset(self):
|
||||
@@ -139,10 +485,10 @@ class SafetyManager:
|
||||
self.daily_trades = 0
|
||||
self.daily_reset_time = now.replace(hour=0, minute=0, second=0)
|
||||
|
||||
# Auto-reset circuit breaker on new day (if triggered by daily limit)
|
||||
if self.trading_halted and 'Daily' in self.halt_reason:
|
||||
# Auto-reset circuit breaker on new day
|
||||
if self.trading_halted:
|
||||
self.reset_circuit_breaker()
|
||||
logger.info("Daily circuit breaker auto-reset on new trading day")
|
||||
logger.info("Circuit breaker auto-reset on new trading day")
|
||||
|
||||
def get_status(self) -> Dict:
|
||||
"""Get safety manager status"""
|
||||
@@ -152,7 +498,18 @@ class SafetyManager:
|
||||
'daily_pnl': round(self.daily_pnl, 2),
|
||||
'daily_trades': self.daily_trades,
|
||||
'peak_equity': round(self.peak_equity, 2),
|
||||
'yesterday_equity': round(self.yesterday_equity, 2),
|
||||
'max_position_pct': self.max_position_pct,
|
||||
'max_daily_loss_pct': self.max_daily_loss_pct,
|
||||
'max_total_loss_pct': self.max_total_loss_pct,
|
||||
'drawdown_stop_active': self.drawdown_stop_active,
|
||||
'volatility_mult': self.current_volatility_mult,
|
||||
'max_concurrent_positions': self.max_concurrent_positions,
|
||||
'max_correlated_positions': self.max_correlated_positions,
|
||||
'max_same_direction': self.max_same_direction,
|
||||
'max_short_positions': self.max_short_positions,
|
||||
'max_total_positions': self.max_total_positions,
|
||||
'yesterday_close_drawdown_limit': self.yesterday_close_drawdown_limit,
|
||||
'stop_loss_default_pct_long': self.stop_loss_default_pct_long,
|
||||
'stop_loss_default_pct_short': self.stop_loss_default_pct_short,
|
||||
}
|
||||
|
||||
@@ -114,6 +114,18 @@ class DataStore:
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS trade_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trade_id INTEGER NOT NULL,
|
||||
event_type TEXT NOT NULL,
|
||||
amount REAL,
|
||||
price REAL,
|
||||
pnl REAL,
|
||||
fees REAL,
|
||||
event_time TEXT NOT NULL,
|
||||
FOREIGN KEY (trade_id) REFERENCES trades(id)
|
||||
);
|
||||
""")
|
||||
c.commit()
|
||||
logger.info(f"Database initialized at {self.db_path}")
|
||||
@@ -262,6 +274,53 @@ class DataStore:
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
def reduce_position(self, trade_id: int, reduce_amount: float,
|
||||
exit_price: float, exit_time: datetime, fees: float = 0):
|
||||
"""
|
||||
Partially close a position by reducing amount.
|
||||
The original trade remains open with reduced amount.
|
||||
Records a partial P&L entry.
|
||||
"""
|
||||
trade = self.conn.execute(
|
||||
"SELECT * FROM trades WHERE id=?", (trade_id,)
|
||||
).fetchone()
|
||||
|
||||
if not trade:
|
||||
return
|
||||
|
||||
trade = dict(trade)
|
||||
|
||||
# Calculate P&L for the reduced portion
|
||||
if trade['side'] == 'buy':
|
||||
pnl = (exit_price - trade['entry_price']) * reduce_amount - fees
|
||||
else:
|
||||
pnl = (trade['entry_price'] - exit_price) * reduce_amount - fees
|
||||
|
||||
remaining_amount = abs(trade['amount']) - reduce_amount
|
||||
|
||||
if remaining_amount <= 0:
|
||||
# Fully close
|
||||
self.close_position(trade_id, exit_price, exit_time, fees)
|
||||
return
|
||||
|
||||
# Update original trade with reduced amount
|
||||
self.conn.execute(
|
||||
"""UPDATE trades SET amount=? WHERE id=?""",
|
||||
(remaining_amount if trade['side'] == 'buy' else -remaining_amount, trade_id)
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
# Record the partial close in a separate table or as a trade event
|
||||
# For now, just record it in trade_events
|
||||
self.conn.execute(
|
||||
"""INSERT INTO trade_events
|
||||
(trade_id, event_type, amount, price, pnl, fees, event_time)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)""",
|
||||
(trade_id, 'partial_close', reduce_amount, exit_price,
|
||||
round(pnl, 4), fees, exit_time.isoformat())
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
# --- Strategy performance ---
|
||||
|
||||
def record_strategy_result(self, strategy_id: str, params: Dict, metrics: Dict):
|
||||
|
||||
+258
-8
@@ -17,6 +17,57 @@ from datetime import datetime, timedelta
|
||||
from loguru import logger
|
||||
import numpy as np
|
||||
|
||||
|
||||
def detect_market_regime(df, lookback: int = 20) -> dict:
|
||||
"""
|
||||
Detect current market regime from recent price data.
|
||||
Returns dict with:
|
||||
'regime': 'trending_up', 'trending_down', 'ranging', 'volatile'
|
||||
'adx': average directional index (trend strength)
|
||||
'volatility_ratio': recent vol vs average vol
|
||||
"""
|
||||
if df is None or len(df) < lookback + 10:
|
||||
return {'regime': 'unknown', 'adx': 0, 'volatility_ratio': 1.0}
|
||||
|
||||
close = df['close'].values[-lookback:]
|
||||
high = df['high'].values[-lookback:]
|
||||
low = df['low'].values[-lookback:]
|
||||
|
||||
# Simple trend: linear regression slope on close prices
|
||||
x = np.arange(len(close))
|
||||
slope = np.polyfit(x, close, 1)[0] if len(close) > 1 else 0
|
||||
slope_pct = slope / np.mean(close) * 100 if np.mean(close) > 0 else 0
|
||||
|
||||
# Volatility: ATR-like measure
|
||||
tr_list = []
|
||||
for i in range(1, len(close)):
|
||||
tr = max(high[i] - low[i], abs(high[i] - close[i-1]), abs(low[i] - close[i-1]))
|
||||
tr_list.append(tr)
|
||||
recent_vol = np.mean(tr_list) if tr_list else 0
|
||||
avg_vol = np.std(close) if len(close) > 1 else 0
|
||||
vol_ratio = recent_vol / avg_vol if avg_vol > 0 else 1.0
|
||||
|
||||
# Range quality: how much does price oscillate vs trending?
|
||||
high_low_range = (np.max(high) - np.min(low)) / np.mean(close) if np.mean(close) > 0 else 0
|
||||
|
||||
# Classify regime
|
||||
if abs(slope_pct) > 0.05: # Significant directional slope
|
||||
if slope_pct > 0:
|
||||
regime = 'trending_up'
|
||||
else:
|
||||
regime = 'trending_down'
|
||||
elif vol_ratio > 2.0:
|
||||
regime = 'volatile'
|
||||
else:
|
||||
regime = 'ranging'
|
||||
|
||||
return {
|
||||
'regime': regime,
|
||||
'slope_pct': round(slope_pct, 4),
|
||||
'volatility_ratio': round(vol_ratio, 2),
|
||||
'trend_strength': round(abs(slope_pct) * 20, 1), # normalized 0-10
|
||||
}
|
||||
|
||||
# Setup logging
|
||||
log_path = Path(__file__).parent.parent / "logs"
|
||||
log_path.mkdir(exist_ok=True)
|
||||
@@ -43,6 +94,7 @@ from ml.genetic import GeneticEvolver, StrategyGenome
|
||||
from strategies.auto_strategy import genome_to_strategy, evaluate_genome_signal
|
||||
from trading.executor import TradingExecutor
|
||||
from core.safety import SafetyManager
|
||||
from core.regime_detector import RegimeDetector, MarketRegime
|
||||
from reporting.telegram_reporter import TelegramReporter
|
||||
from reporting.krystie_bridge import KrystieBridge
|
||||
|
||||
@@ -110,6 +162,12 @@ class BiggFishAuto:
|
||||
self.config['ga'], self.backtest_engine, self.store
|
||||
)
|
||||
|
||||
# Market Regime Detector (shared across all symbols)
|
||||
regime_config = self.config.get('regime', {})
|
||||
self.regime_detector = RegimeDetector(regime_config)
|
||||
self.symbol_regimes: Dict[str, Dict] = {} # Per-symbol regime cache
|
||||
self.last_regime_log: datetime = datetime.min
|
||||
|
||||
# Telegram daily reporter
|
||||
tg_config = self.config.get('telegram', {})
|
||||
if tg_config.get('enabled') and tg_config.get('bot_token') and tg_config.get('chat_id'):
|
||||
@@ -281,11 +339,14 @@ class BiggFishAuto:
|
||||
return self.executor
|
||||
|
||||
def _trading_cycle(self):
|
||||
"""Run one trading cycle across all markets"""
|
||||
"""Run one trading cycle across all markets with regime-aware position management"""
|
||||
self.cycle_count += 1
|
||||
all_symbols = self._get_tradeable_symbols()
|
||||
|
||||
# Check exits first
|
||||
# ── STEP 1: Detect market regime for each symbol ─────────────────────────
|
||||
self._detect_regimes(all_symbols)
|
||||
|
||||
# ── STEP 2: Check exits (normal stop/take-profit exits) ─────────────────
|
||||
try:
|
||||
current_prices = {}
|
||||
for symbol in all_symbols:
|
||||
@@ -294,7 +355,6 @@ class BiggFishAuto:
|
||||
if price:
|
||||
current_prices[symbol] = price
|
||||
|
||||
# Check exits per executor
|
||||
stock_prices = {s: p for s, p in current_prices.items() if not self._is_forex(s)}
|
||||
forex_prices = {s: p for s, p in current_prices.items() if self._is_forex(s)}
|
||||
|
||||
@@ -316,12 +376,37 @@ class BiggFishAuto:
|
||||
except Exception as e:
|
||||
logger.error(f"Exit check error: {e}")
|
||||
|
||||
# Check safety
|
||||
# ── STEP 3: Regime-aware forced exits ───────────────────────────────────
|
||||
# Force-close positions that are fighting the regime
|
||||
self._regime_force_close(all_symbols, current_prices)
|
||||
|
||||
# ── STEP 4: Check safety ────────────────────────────────────────────────
|
||||
if not self.safety.is_trading_allowed():
|
||||
logger.warning(f"Trading halted: {self.safety.halt_reason}")
|
||||
return
|
||||
|
||||
# Evaluate each symbol
|
||||
# ── STEP 4b: BROKER POSITION SYNC CHECK ─────────────────────────────────
|
||||
# Validate that store and broker agree on positions.
|
||||
# If broker has more positions than store (e.g., manual trades), block new trades.
|
||||
try:
|
||||
broker_pos = self.broker.get_positions()
|
||||
store_pos = self.store.get_open_positions()
|
||||
broker_ok, broker_msg = self.safety.validate_broker_position_count(
|
||||
broker_pos, store_pos
|
||||
)
|
||||
if not broker_ok:
|
||||
logger.warning(f"BROKER POSITION BLOCK: {broker_msg}")
|
||||
return # Block all new trades until positions are reduced
|
||||
except Exception as e:
|
||||
logger.error(f"Broker position sync check failed: {e}")
|
||||
|
||||
# ── STEP 5: Log regime status periodically ──────────────────────────────
|
||||
now = datetime.utcnow()
|
||||
if (now - self.last_regime_log).total_seconds() > 300: # Every 5 min
|
||||
self._log_regime_status(all_symbols)
|
||||
self.last_regime_log = now
|
||||
|
||||
# ── STEP 6: Evaluate each symbol ─────────────────────────────────────────
|
||||
for symbol in all_symbols:
|
||||
try:
|
||||
self._trade_symbol(symbol)
|
||||
@@ -331,6 +416,145 @@ class BiggFishAuto:
|
||||
# Keep only last 20 trades
|
||||
self.recent_trades = self.recent_trades[-20:]
|
||||
|
||||
def _detect_regimes(self, symbols: list):
|
||||
"""Detect market regime for each symbol using 1h data (more reliable for regime)."""
|
||||
for symbol in symbols:
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '1h',
|
||||
start=datetime.utcnow() - timedelta(days=30)
|
||||
)
|
||||
if df is None or len(df) < 100:
|
||||
# Fall back to 5m
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '5m',
|
||||
start=datetime.utcnow() - timedelta(days=7)
|
||||
)
|
||||
if df is None or len(df) < 60:
|
||||
self.symbol_regimes[symbol] = {
|
||||
'regime': MarketRegime.UNKNOWN,
|
||||
'confidence': 0.0,
|
||||
'trend_strength': 0.0,
|
||||
'volatility_ratio': 1.0,
|
||||
'bb_position': 0.5,
|
||||
'regime_change': False,
|
||||
'early_warning': False,
|
||||
'warning_message': None,
|
||||
'indicators': {},
|
||||
'adaptive_params': self.regime_detector.get_adaptive_params(MarketRegime.UNKNOWN),
|
||||
}
|
||||
continue
|
||||
|
||||
regime_data = self.regime_detector.detect_regime(df, symbol)
|
||||
adaptive = self.regime_detector.get_adaptive_params(regime_data['regime'])
|
||||
regime_data['adaptive_params'] = adaptive
|
||||
self.symbol_regimes[symbol] = regime_data
|
||||
|
||||
# Emit early warning to logs
|
||||
if regime_data.get('early_warning'):
|
||||
logger.warning(
|
||||
f"⚠️ REGIME WARNING [{symbol}]: {regime_data.get('warning_message', '')} "
|
||||
f"| Regime: {regime_data['regime']} | ADX: {regime_data['trend_strength']:.1f}"
|
||||
)
|
||||
|
||||
# Emit regime change to logs
|
||||
if regime_data.get('regime_change'):
|
||||
logger.warning(
|
||||
f"🔄 REGIME SHIFT [{symbol}]: {regime_data['regime_change_type']} "
|
||||
f"| {adaptive['regime_label']}"
|
||||
)
|
||||
|
||||
def _regime_force_close(self, symbols: list, current_prices: dict):
|
||||
"""
|
||||
Force-close positions that are counter to the current regime.
|
||||
E.g., in a strong uptrend, close shorts. In ranging, close all directional.
|
||||
"""
|
||||
try:
|
||||
open_positions = self.store.get_open_positions()
|
||||
if not open_positions:
|
||||
return
|
||||
|
||||
for position in open_positions:
|
||||
symbol = position['symbol']
|
||||
if symbol not in symbols:
|
||||
continue
|
||||
|
||||
regime_info = self.symbol_regimes.get(symbol, {})
|
||||
regime = regime_info.get('regime', MarketRegime.UNKNOWN)
|
||||
params = regime_info.get('adaptive_params', {})
|
||||
is_short = position.get('metadata', {}).get('direction') == 'short'
|
||||
reduction = params.get('short_position_reduction', 0.0)
|
||||
price = current_prices.get(symbol)
|
||||
|
||||
if not price:
|
||||
continue
|
||||
|
||||
force_close = False
|
||||
close_reason = ''
|
||||
|
||||
if reduction >= 1.0 and is_short:
|
||||
# Ranging or strong uptrend: close all shorts
|
||||
force_close = True
|
||||
close_reason = f"regime_{regime}_close_short"
|
||||
elif reduction >= 0.5 and reduction < 1.0 and is_short:
|
||||
# Moderate regime conflict: close 50%
|
||||
force_close = True
|
||||
close_reason = f"regime_{regime}_reduce_short"
|
||||
|
||||
if force_close:
|
||||
try:
|
||||
close_side = 'buy' # Buy to cover short
|
||||
close_amount = position['amount']
|
||||
if reduction < 1.0:
|
||||
close_amount = round(position['amount'] * reduction, 4)
|
||||
|
||||
self.broker.place_market_order(symbol, close_amount, close_side)
|
||||
pnl = (position['entry_price'] - price) * close_amount
|
||||
self.store.close_position(
|
||||
position['id'], price, datetime.utcnow(),
|
||||
fees=close_amount * price * self.executor.commission_rate
|
||||
)
|
||||
self.safety.record_trade_result(pnl)
|
||||
logger.warning(
|
||||
f"🚨 REGIME FORCE-CLOSE [{symbol}]: {close_reason} "
|
||||
f"| {close_amount} @ ${price:.4f} | P&L: ${pnl:+.2f}"
|
||||
)
|
||||
self.recent_trades.append({
|
||||
'symbol': symbol,
|
||||
'side': close_side,
|
||||
'amount': close_amount,
|
||||
'exit_price': price,
|
||||
'pnl': round(pnl, 2),
|
||||
'exit_reason': close_reason,
|
||||
'direction': 'short',
|
||||
})
|
||||
except Exception as e:
|
||||
logger.error(f"Regime force-close error for {symbol}: {e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Regime force-close error: {e}")
|
||||
|
||||
def _log_regime_status(self, symbols: list):
|
||||
"""Log current regime status for all symbols (called every 5 min)."""
|
||||
lines = ["[REGIME STATUS]"]
|
||||
for symbol in symbols[:6]: # Limit to 6 symbols for dashboard
|
||||
info = self.symbol_regimes.get(symbol, {})
|
||||
regime = info.get('regime', MarketRegime.UNKNOWN)
|
||||
params = info.get('adaptive_params', {})
|
||||
adx = info.get('trend_strength', 0)
|
||||
vol = info.get('volatility_ratio', 1.0)
|
||||
short_red = params.get('short_position_reduction', 0.0)
|
||||
pos_mult = params.get('position_size_multiplier', 1.0)
|
||||
emoji = '📈' if regime == MarketRegime.TRENDING_UP else (
|
||||
'📉' if regime == MarketRegime.TRENDING_DOWN else (
|
||||
'🔄' if regime == MarketRegime.RANGING else (
|
||||
'💥' if regime == MarketRegime.VOLATILE else '❓')))
|
||||
lines.append(
|
||||
f" {emoji} {symbol}: {regime[:12]:12s} | "
|
||||
f"ADX={adx:4.1f} | Vol={vol:.1f}x | "
|
||||
f"Size={pos_mult:.0%} | ShortReduce={short_red:.0%}"
|
||||
)
|
||||
logger.info("\n".join(lines))
|
||||
|
||||
def _get_tradeable_symbols(self) -> list:
|
||||
"""Get symbols that can be traded right now"""
|
||||
stock_symbols = self.config['trading'].get('symbols', [])
|
||||
@@ -374,6 +598,18 @@ class BiggFishAuto:
|
||||
# Get market state
|
||||
state = self.feature_engine.get_state_vector(features_df, -1)
|
||||
|
||||
# Detect market regime for adaptive decision-making
|
||||
regime = detect_market_regime(df, lookback=20)
|
||||
regime_signal_mult = 1.0
|
||||
# In ranging markets, be more conservative with short signals
|
||||
# In trending markets, follow the trend more aggressively
|
||||
if regime['regime'] == 'ranging':
|
||||
regime_signal_mult = 0.7 # Reduce confidence in ranging
|
||||
elif regime['regime'] == 'trending_down':
|
||||
regime_signal_mult = 1.2 # Boost short confidence in downtrend
|
||||
elif regime['regime'] == 'trending_up':
|
||||
regime_signal_mult = 1.2 # Boost long confidence in uptrend
|
||||
|
||||
# Get GA signal
|
||||
best_genome = self.ga_evolver.get_best_genome()
|
||||
ga_signal = {'signal': 'hold', 'confidence': 0.0}
|
||||
@@ -383,6 +619,9 @@ class BiggFishAuto:
|
||||
raw_df = self.feature_engine.compute(df)
|
||||
if raw_df is not None and len(raw_df) > 0:
|
||||
ga_signal = evaluate_genome_signal(best_genome, raw_df, len(raw_df) - 1)
|
||||
# Scale confidence by regime
|
||||
base_conf = ga_signal.get('confidence', 0.0)
|
||||
regime_adjusted_conf = min(base_conf * regime_signal_mult, 1.0)
|
||||
strategy_params = {
|
||||
'stop_loss': ga_signal.get('stop_loss'),
|
||||
'take_profit': ga_signal.get('take_profit'),
|
||||
@@ -392,17 +631,23 @@ class BiggFishAuto:
|
||||
'strategy_id': f"ga_gen{best_genome.generation}",
|
||||
}
|
||||
|
||||
# Augment state with GA signal + portfolio
|
||||
# Augment state with GA signal + portfolio + regime
|
||||
portfolio_state = executor.get_portfolio_state()
|
||||
signal_dir = 1.0 if ga_signal['signal'] == 'buy' else (
|
||||
-1.0 if ga_signal['signal'] == 'sell' else 0.0)
|
||||
|
||||
# Encode regime as numeric (for RL state)
|
||||
regime_code = {'trending_up': 1.0, 'trending_down': -1.0,
|
||||
'ranging': 0.0, 'volatile': 0.5,
|
||||
'unknown': 0.0}.get(regime['regime'], 0.0)
|
||||
|
||||
augmented_state = np.concatenate([
|
||||
state,
|
||||
[portfolio_state.get('position_ratio', 0),
|
||||
portfolio_state.get('unrealized_pnl', 0),
|
||||
portfolio_state.get('time_in_position', 0)],
|
||||
[signal_dir, ga_signal.get('confidence', 0.0)],
|
||||
[signal_dir, ga_signal.get('confidence', 0.0) * regime_signal_mult],
|
||||
[regime_code, regime.get('trend_strength', 0) / 10.0],
|
||||
])
|
||||
|
||||
# RL agent decides (now with 7 actions including shorts)
|
||||
@@ -416,9 +661,14 @@ class BiggFishAuto:
|
||||
if not current_price:
|
||||
return
|
||||
|
||||
# Get combined equity across all brokers for safety checks
|
||||
combined = self._get_combined_portfolio()
|
||||
combined_equity = combined['equity']
|
||||
|
||||
# Execute
|
||||
trade = executor.execute_signal(
|
||||
symbol, action, current_price, strategy_params
|
||||
symbol, action, current_price, strategy_params,
|
||||
combined_equity=combined_equity
|
||||
)
|
||||
|
||||
if trade:
|
||||
|
||||
@@ -0,0 +1,904 @@
|
||||
"""
|
||||
BIGGFISH Autonomous Self-Learning Trading Bot
|
||||
24/7 entry point - runs forever, learns continuously, trades autonomously.
|
||||
|
||||
Usage:
|
||||
python -m src.main_auto
|
||||
python -m src.main_auto --config config/auto_config.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import threading
|
||||
import signal as signal_module
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from loguru import logger
|
||||
import numpy as np
|
||||
|
||||
# Setup logging
|
||||
log_path = Path(__file__).parent.parent / "logs"
|
||||
log_path.mkdir(exist_ok=True)
|
||||
logger.add(
|
||||
log_path / "biggfish_auto_{time}.log",
|
||||
rotation="1 day",
|
||||
retention="30 days",
|
||||
level="INFO"
|
||||
)
|
||||
|
||||
from trading.broker import AlpacaBroker
|
||||
from data.store import DataStore
|
||||
from data.candle_cache import CandleCache
|
||||
try:
|
||||
from trading.oanda_broker import OandaBroker
|
||||
except ImportError:
|
||||
OandaBroker = None
|
||||
from data.features import FeatureEngine
|
||||
from backtest.engine import BacktestEngine
|
||||
from backtest.metrics import compute_metrics
|
||||
from ml.rl_agent import RLAgent
|
||||
from ml.rl_environment import TradingEnvironment
|
||||
from ml.genetic import GeneticEvolver, StrategyGenome
|
||||
from strategies.auto_strategy import genome_to_strategy, evaluate_genome_signal
|
||||
from trading.executor import TradingExecutor
|
||||
from core.safety import SafetyManager
|
||||
from reporting.telegram_reporter import TelegramReporter
|
||||
from reporting.krystie_bridge import KrystieBridge
|
||||
|
||||
|
||||
class BiggFishAuto:
|
||||
"""
|
||||
24/7 Autonomous self-learning trading bot.
|
||||
Coordinates trading, backtesting, RL training, and GA evolution.
|
||||
"""
|
||||
|
||||
def __init__(self, config_path: str = "config/auto_config.json"):
|
||||
logger.info("Initializing BIGGFISH Autonomous Trader...")
|
||||
|
||||
# Load config
|
||||
with open(config_path) as f:
|
||||
self.config = json.load(f)
|
||||
|
||||
# Initialize components
|
||||
self.store = DataStore(self.config['database']['path'])
|
||||
self.store.initialize()
|
||||
|
||||
self.broker = AlpacaBroker(self.config['alpaca'])
|
||||
self.feature_engine = FeatureEngine()
|
||||
|
||||
# OANDA broker for forex (optional)
|
||||
self.oanda_broker = None
|
||||
oanda_config = self.config.get('oanda')
|
||||
if oanda_config and oanda_config.get('api_token') and OandaBroker:
|
||||
try:
|
||||
self.oanda_broker = OandaBroker(oanda_config)
|
||||
logger.info("OANDA forex broker connected")
|
||||
except Exception as e:
|
||||
logger.warning(f"OANDA connection failed (forex disabled): {e}")
|
||||
|
||||
self.candle_cache = CandleCache(self.store, oanda_broker=self.oanda_broker)
|
||||
|
||||
self.backtest_engine = BacktestEngine(
|
||||
initial_capital=self.config['backtest']['initial_capital'],
|
||||
commission_rate=self.config['trading'].get('commission_rate', 0.001)
|
||||
)
|
||||
|
||||
self.safety = SafetyManager(self.config['safety'], self.store)
|
||||
|
||||
# Executors per broker
|
||||
self.executor = TradingExecutor(
|
||||
self.broker, self.store, self.safety, self.config['trading']
|
||||
)
|
||||
self.forex_executor = None
|
||||
if self.oanda_broker:
|
||||
self.forex_executor = TradingExecutor(
|
||||
self.oanda_broker, self.store, self.safety, self.config['trading']
|
||||
)
|
||||
|
||||
# RL Agent
|
||||
env_temp = TradingEnvironment(self.feature_engine,
|
||||
initial_capital=self.config['trading']['initial_capital'])
|
||||
self.rl_agent = RLAgent(
|
||||
state_dim=env_temp.state_dim,
|
||||
action_dim=TradingEnvironment.NUM_ACTIONS,
|
||||
config=self.config['rl']
|
||||
)
|
||||
|
||||
# GA Evolver
|
||||
self.ga_evolver = GeneticEvolver(
|
||||
self.config['ga'], self.backtest_engine, self.store
|
||||
)
|
||||
|
||||
# Telegram daily reporter
|
||||
tg_config = self.config.get('telegram', {})
|
||||
if tg_config.get('enabled') and tg_config.get('bot_token') and tg_config.get('chat_id'):
|
||||
self.telegram = TelegramReporter(tg_config['bot_token'], tg_config['chat_id'])
|
||||
logger.info("Telegram reporting enabled")
|
||||
else:
|
||||
self.telegram = None
|
||||
logger.info("Telegram reporting disabled (no config)")
|
||||
|
||||
# Krystie bridge (writes status/events to JSON for Krystie to read)
|
||||
self.krystie = KrystieBridge(self.config['database']['path'].rsplit('/', 1)[0] or 'data')
|
||||
|
||||
# State
|
||||
self.running = False
|
||||
self.start_time = None
|
||||
self.cycle_count = 0
|
||||
self.last_backtest = datetime.min
|
||||
self.last_rl_train = datetime.min
|
||||
self.last_ga_evolve = datetime.min
|
||||
self.last_dashboard = datetime.min
|
||||
self.last_daily_report = datetime.min
|
||||
self.recent_trades = [] # Last 10 trades for dashboard
|
||||
self.backtest_results = {} # symbol -> {'sharpe': x, 'win_rate': y}
|
||||
|
||||
logger.info("BIGGFISH Autonomous Trader initialized")
|
||||
|
||||
def start(self):
|
||||
"""Start the autonomous trading system"""
|
||||
self.running = True
|
||||
self.start_time = datetime.utcnow()
|
||||
|
||||
# Register shutdown handler
|
||||
signal_module.signal(signal_module.SIGINT, self._signal_handler)
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info(" BIGGFISH AUTONOMOUS TRADER STARTING")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# 1. Load saved state
|
||||
self._load_state()
|
||||
|
||||
self.krystie.log_startup()
|
||||
|
||||
# 1.5. Initialize peak equity from current portfolio value
|
||||
try:
|
||||
portfolio = self.broker.get_portfolio()
|
||||
current_equity = portfolio.get('portfolio_value', self.config['safety']['initial_capital'])
|
||||
self.safety.peak_equity = max(current_equity, self.safety.peak_equity)
|
||||
logger.info(f"Initialized peak equity: ${self.safety.peak_equity:,.2f}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not initialize peak equity: {e}")
|
||||
|
||||
# 2. Warm candle cache
|
||||
self._warm_cache()
|
||||
|
||||
# 3. Initial GA population
|
||||
self.ga_evolver.initialize_population()
|
||||
if not self.ga_evolver.population:
|
||||
logger.info("Running initial GA evolution...")
|
||||
self._run_ga_evolution()
|
||||
|
||||
# 4. Initial RL training if no checkpoint
|
||||
if self.rl_agent.steps == 0:
|
||||
logger.info("Running initial RL training...")
|
||||
self._run_rl_training()
|
||||
|
||||
# 5. Main loop
|
||||
logger.info("Entering main loop...")
|
||||
self._print_dashboard()
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
cycle_start = datetime.utcnow()
|
||||
|
||||
# Trading cycle (run if any market is open)
|
||||
any_market_open = self.broker.is_market_open()
|
||||
if self.oanda_broker:
|
||||
any_market_open = any_market_open or self.oanda_broker.is_market_open()
|
||||
|
||||
if any_market_open:
|
||||
self._trading_cycle()
|
||||
else:
|
||||
logger.debug("All markets closed - running learning tasks")
|
||||
|
||||
# Periodic tasks (run regardless of market hours)
|
||||
now = datetime.utcnow()
|
||||
|
||||
# Cache update (every 5 min)
|
||||
cache_interval = self.config['cache']['update_interval_seconds']
|
||||
if (now - self.last_backtest).total_seconds() > cache_interval:
|
||||
self._update_cache()
|
||||
|
||||
# Backtest (every 30 min)
|
||||
bt_interval = self.config['backtest']['interval_seconds']
|
||||
if (now - self.last_backtest).total_seconds() > bt_interval:
|
||||
self._run_backtest()
|
||||
self.last_backtest = now
|
||||
|
||||
# RL training (every 2 hours)
|
||||
rl_interval = self.config['rl']['train_interval_hours'] * 3600
|
||||
if (now - self.last_rl_train).total_seconds() > rl_interval:
|
||||
self._run_rl_training()
|
||||
self.last_rl_train = now
|
||||
|
||||
# GA evolution (every 6 hours)
|
||||
ga_interval = self.config['ga']['evolution_interval_hours'] * 3600
|
||||
if (now - self.last_ga_evolve).total_seconds() > ga_interval:
|
||||
self._run_ga_evolution()
|
||||
self.last_ga_evolve = now
|
||||
|
||||
# Daily Telegram report (once per day, around market close ~21:00 UTC)
|
||||
if self.telegram and self._should_send_daily_report(now):
|
||||
self._send_daily_telegram_report()
|
||||
self.last_daily_report = now
|
||||
|
||||
# Dashboard (every minute)
|
||||
dash_interval = self.config['reporting']['dashboard_interval_seconds']
|
||||
if (now - self.last_dashboard).total_seconds() > dash_interval:
|
||||
self._print_dashboard()
|
||||
self.last_dashboard = now
|
||||
|
||||
# Sleep until next cycle
|
||||
elapsed = (datetime.utcnow() - cycle_start).total_seconds()
|
||||
sleep_time = max(1, self.config['trading']['cycle_interval_seconds'] - elapsed)
|
||||
time.sleep(sleep_time)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
self._shutdown()
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Main loop error: {e}", exc_info=True)
|
||||
time.sleep(30)
|
||||
|
||||
def _is_forex(self, symbol: str) -> bool:
|
||||
"""Check if symbol is a forex pair (OANDA format: XXX_YYY)"""
|
||||
return '_' in symbol and len(symbol) == 7
|
||||
|
||||
def _get_broker(self, symbol: str):
|
||||
"""Get the appropriate broker for a symbol"""
|
||||
if self._is_forex(symbol) and self.oanda_broker:
|
||||
return self.oanda_broker
|
||||
return self.broker
|
||||
|
||||
def _get_executor(self, symbol: str):
|
||||
"""Get the appropriate executor for a symbol"""
|
||||
if self._is_forex(symbol) and self.forex_executor:
|
||||
return self.forex_executor
|
||||
return self.executor
|
||||
|
||||
def _trading_cycle(self):
|
||||
"""Run one trading cycle across all markets"""
|
||||
self.cycle_count += 1
|
||||
all_symbols = self._get_tradeable_symbols()
|
||||
|
||||
# Check exits first
|
||||
try:
|
||||
current_prices = {}
|
||||
for symbol in all_symbols:
|
||||
broker = self._get_broker(symbol)
|
||||
price = broker.get_latest_price(symbol)
|
||||
if price:
|
||||
current_prices[symbol] = price
|
||||
|
||||
# Check exits per executor
|
||||
stock_prices = {s: p for s, p in current_prices.items() if not self._is_forex(s)}
|
||||
forex_prices = {s: p for s, p in current_prices.items() if self._is_forex(s)}
|
||||
|
||||
if stock_prices:
|
||||
closed = self.executor.check_exits(stock_prices)
|
||||
for trade in closed:
|
||||
logger.info(f"CLOSED {trade['symbol']}: ${trade['pnl']:+.2f} "
|
||||
f"({trade['pnl_pct']:+.1f}%) [{trade['exit_reason']}]")
|
||||
self.recent_trades.append(trade)
|
||||
self.krystie.log_trade(trade)
|
||||
|
||||
if forex_prices and self.forex_executor:
|
||||
closed = self.forex_executor.check_exits(forex_prices)
|
||||
for trade in closed:
|
||||
logger.info(f"CLOSED {trade['symbol']}: ${trade['pnl']:+.2f} "
|
||||
f"({trade['pnl_pct']:+.1f}%) [{trade['exit_reason']}]")
|
||||
self.recent_trades.append(trade)
|
||||
self.krystie.log_trade(trade)
|
||||
except Exception as e:
|
||||
logger.error(f"Exit check error: {e}")
|
||||
|
||||
# Get combined portfolio equity for safety checks
|
||||
try:
|
||||
alpaca_portfolio = self.broker.get_portfolio()
|
||||
combined_equity = alpaca_portfolio['equity']
|
||||
if self.oanda_broker:
|
||||
oanda_portfolio = self.oanda_broker.get_portfolio()
|
||||
combined_equity += oanda_portfolio['equity']
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting combined portfolio: {e}")
|
||||
combined_equity = None
|
||||
|
||||
# Check safety
|
||||
if not self.safety.is_trading_allowed():
|
||||
logger.warning(f"Trading halted: {self.safety.halt_reason}")
|
||||
return
|
||||
|
||||
# Evaluate each symbol
|
||||
for symbol in all_symbols:
|
||||
try:
|
||||
self._trade_symbol(symbol, combined_equity)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error trading {symbol}: {e}")
|
||||
|
||||
# Keep only last 20 trades
|
||||
self.recent_trades = self.recent_trades[-20:]
|
||||
|
||||
def _get_tradeable_symbols(self) -> list:
|
||||
"""Get symbols that can be traded right now"""
|
||||
stock_symbols = self.config['trading'].get('symbols', [])
|
||||
forex_symbols = self.config['trading'].get('forex_symbols', [])
|
||||
|
||||
tradeable = []
|
||||
|
||||
# Stocks: only if Alpaca market is open
|
||||
if self.broker.is_market_open():
|
||||
tradeable.extend(stock_symbols)
|
||||
|
||||
# Forex: if OANDA is connected and forex market is open (24/5)
|
||||
if self.oanda_broker and self.oanda_broker.is_market_open():
|
||||
tradeable.extend(forex_symbols)
|
||||
|
||||
return tradeable
|
||||
|
||||
def _trade_symbol(self, symbol: str, combined_equity: float = None):
|
||||
"""Evaluate and potentially trade a single symbol (scalping mode)"""
|
||||
# Filter by backtest Sharpe ratio
|
||||
min_sharpe = self.config['trading'].get('min_backtest_sharpe', 0.5)
|
||||
if symbol in self.backtest_results:
|
||||
if self.backtest_results[symbol]['sharpe'] < min_sharpe:
|
||||
logger.debug(f"Skipping {symbol}: Sharpe {self.backtest_results[symbol]['sharpe']:.2f} < {min_sharpe}")
|
||||
return
|
||||
|
||||
broker = self._get_broker(symbol)
|
||||
executor = self._get_executor(symbol)
|
||||
|
||||
# Use 5m candles for scalping, fall back to 1h
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '5m',
|
||||
start=datetime.utcnow() - timedelta(days=14)
|
||||
)
|
||||
if df is None or len(df) < 60:
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '1h',
|
||||
start=datetime.utcnow() - timedelta(days=14)
|
||||
)
|
||||
if df is None or len(df) < 60:
|
||||
return
|
||||
|
||||
# Compute features
|
||||
features_df = self.feature_engine.compute_and_normalize(df)
|
||||
if features_df is None or len(features_df) < 10:
|
||||
return
|
||||
|
||||
# Get market state
|
||||
state = self.feature_engine.get_state_vector(features_df, -1)
|
||||
|
||||
# Get GA signal
|
||||
best_genome = self.ga_evolver.get_best_genome()
|
||||
ga_signal = {'signal': 'hold', 'confidence': 0.0}
|
||||
strategy_params = {}
|
||||
|
||||
if best_genome:
|
||||
raw_df = self.feature_engine.compute(df)
|
||||
if raw_df is not None and len(raw_df) > 0:
|
||||
ga_signal = evaluate_genome_signal(best_genome, raw_df, len(raw_df) - 1)
|
||||
|
||||
# Use config defaults if GA doesn't provide stop-loss/take-profit
|
||||
sl_pct = self.config['trading'].get('stop_loss_pct', 2.5) / 100
|
||||
tp_pct = self.config['trading'].get('take_profit_pct', 5.0) / 100
|
||||
|
||||
# CRITICAL FIX: Convert percentages to absolute prices
|
||||
# Stop-loss BELOW current price for longs, ABOVE for shorts
|
||||
# Take-profit ABOVE current price for longs, BELOW for shorts
|
||||
current_price = broker.get_latest_price(symbol) or 0
|
||||
|
||||
sl_price = current_price * (1 - sl_pct) if current_price > 0 else None
|
||||
tp_price = current_price * (1 + tp_pct) if current_price > 0 else None
|
||||
short_sl_price = current_price * (1 + sl_pct) if current_price > 0 else None # ABOVE for shorts
|
||||
short_tp_price = current_price * (1 - tp_pct) if current_price > 0 else None # BELOW for shorts
|
||||
|
||||
strategy_params = {
|
||||
'stop_loss': sl_price,
|
||||
'take_profit': tp_price,
|
||||
'short_stop_loss': short_sl_price,
|
||||
'short_take_profit': short_tp_price,
|
||||
'position_pct': ga_signal.get('position_pct', 0.1),
|
||||
'strategy_id': f"ga_gen{best_genome.generation}",
|
||||
}
|
||||
|
||||
# Augment state with GA signal + portfolio
|
||||
portfolio_state = executor.get_portfolio_state()
|
||||
signal_dir = 1.0 if ga_signal['signal'] == 'buy' else (
|
||||
-1.0 if ga_signal['signal'] == 'sell' else 0.0)
|
||||
|
||||
augmented_state = np.concatenate([
|
||||
state,
|
||||
[portfolio_state.get('position_ratio', 0),
|
||||
portfolio_state.get('unrealized_pnl', 0),
|
||||
portfolio_state.get('time_in_position', 0)],
|
||||
[signal_dir, ga_signal.get('confidence', 0.0)],
|
||||
])
|
||||
|
||||
# RL agent decides (now with 7 actions including shorts)
|
||||
action = self.rl_agent.select_action(augmented_state, live_mode=True)
|
||||
|
||||
if action == 0: # Hold
|
||||
return
|
||||
|
||||
# Get current price
|
||||
current_price = broker.get_latest_price(symbol)
|
||||
if not current_price:
|
||||
return
|
||||
|
||||
# Execute (pass combined_equity for multi-broker safety checks)
|
||||
trade = executor.execute_signal(
|
||||
symbol, action, current_price, strategy_params, combined_equity
|
||||
)
|
||||
|
||||
if trade:
|
||||
action_name = TradingEnvironment.ACTION_NAMES[action]
|
||||
logger.info(f"TRADE: {action_name} {symbol} @ ${current_price:.4f}")
|
||||
self.recent_trades.append(trade)
|
||||
self.krystie.log_trade(trade)
|
||||
|
||||
def _all_symbols(self) -> list:
|
||||
"""Get all configured symbols (stocks + forex)"""
|
||||
symbols = list(self.config['trading'].get('symbols', []))
|
||||
symbols.extend(self.config['trading'].get('forex_symbols', []))
|
||||
return symbols
|
||||
|
||||
def _warm_cache(self):
|
||||
"""Warm the candle cache"""
|
||||
logger.info("Warming candle cache...")
|
||||
symbols = self._all_symbols()
|
||||
timeframes = self.config['cache'].get('timeframes', ['1h'])
|
||||
lookback = self.config['cache'].get('warmup_lookback_days', 90)
|
||||
|
||||
self.candle_cache.warm_cache(symbols, timeframes, lookback)
|
||||
logger.info("Cache warmup complete")
|
||||
|
||||
def _update_cache(self):
|
||||
"""Update candle cache incrementally"""
|
||||
symbols = self._all_symbols()
|
||||
timeframes = self.config['cache'].get('timeframes', ['1h'])
|
||||
self.candle_cache.update_cache(symbols, timeframes)
|
||||
|
||||
def _run_backtest(self):
|
||||
"""Run background backtest of current strategy (scalping timeframe)"""
|
||||
best_genome = self.ga_evolver.get_best_genome()
|
||||
if not best_genome:
|
||||
return
|
||||
|
||||
strategy_fn = genome_to_strategy(best_genome)
|
||||
symbols = self._all_symbols()[:5] # Top 5 for breadth
|
||||
|
||||
lookback = self.config['backtest'].get('lookback_days', 14)
|
||||
|
||||
for symbol in symbols:
|
||||
# Prefer 5m data for scalping backtest
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '5m',
|
||||
start=datetime.utcnow() - timedelta(days=lookback)
|
||||
)
|
||||
if df is None or len(df) < 60:
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '1h',
|
||||
start=datetime.utcnow() - timedelta(days=lookback)
|
||||
)
|
||||
if df is None or len(df) < 60:
|
||||
continue
|
||||
|
||||
featured_df = self.feature_engine.compute(df)
|
||||
if featured_df is None or len(featured_df) < 50:
|
||||
continue
|
||||
|
||||
result = self.backtest_engine.run(
|
||||
strategy_fn, featured_df,
|
||||
params=best_genome.to_dict(), symbol=symbol
|
||||
)
|
||||
|
||||
if result.metrics.get('total_trades', 0) > 0:
|
||||
self.store.record_strategy_result(
|
||||
strategy_id=f"ga_gen{best_genome.generation}",
|
||||
params=best_genome.to_dict(),
|
||||
metrics=result.metrics
|
||||
)
|
||||
# Store backtest result for filtering
|
||||
self.backtest_results[symbol] = {
|
||||
'sharpe': result.metrics['sharpe_ratio'],
|
||||
'win_rate': result.metrics['win_rate'],
|
||||
'total_return': result.metrics['total_return'],
|
||||
}
|
||||
logger.info(f"Backtest {symbol}: Sharpe={result.metrics['sharpe_ratio']:.2f} "
|
||||
f"WR={result.metrics['win_rate']:.0f}% "
|
||||
f"Return={result.metrics['total_return']:.1f}%")
|
||||
|
||||
def _run_rl_training(self):
|
||||
"""Run RL batch training"""
|
||||
logger.info("Starting RL training...")
|
||||
env = TradingEnvironment(
|
||||
self.feature_engine,
|
||||
initial_capital=self.config['trading']['initial_capital']
|
||||
)
|
||||
|
||||
total_metrics = {'avg_loss': 0, 'episodes': 0}
|
||||
|
||||
for symbol in self._all_symbols()[:5]:
|
||||
# Prefer 5m data for RL training (more scalping episodes)
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '5m',
|
||||
start=datetime.utcnow() - timedelta(days=14)
|
||||
)
|
||||
if df is None or len(df) < 100:
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '1h',
|
||||
start=datetime.utcnow() - timedelta(days=14)
|
||||
)
|
||||
if df is None or len(df) < 100:
|
||||
continue
|
||||
|
||||
# Create GA signal function for this data
|
||||
best_genome = self.ga_evolver.get_best_genome()
|
||||
ga_fn = None
|
||||
if best_genome:
|
||||
featured = self.feature_engine.compute(df)
|
||||
if featured is not None and len(featured) > 0:
|
||||
def make_ga_fn(genome, feat_df):
|
||||
def fn(step):
|
||||
if step < len(feat_df):
|
||||
return evaluate_genome_signal(genome, feat_df, step)
|
||||
return {'signal': 'hold', 'confidence': 0.0}
|
||||
return fn
|
||||
ga_fn = make_ga_fn(best_genome, featured)
|
||||
|
||||
metrics = self.rl_agent.train_on_episode(env, df, ga_signal_fn=ga_fn)
|
||||
total_metrics['avg_loss'] += metrics.get('avg_loss', 0)
|
||||
total_metrics['episodes'] += 1
|
||||
|
||||
# Save checkpoint
|
||||
self.rl_agent.save(self.store)
|
||||
|
||||
if total_metrics['episodes'] > 0:
|
||||
avg = total_metrics['avg_loss'] / total_metrics['episodes']
|
||||
logger.info(f"RL training complete: {total_metrics['episodes']} episodes, "
|
||||
f"avg_loss={avg:.6f}, epsilon={self.rl_agent.epsilon:.4f}")
|
||||
|
||||
def _run_ga_evolution(self):
|
||||
"""Run GA evolution cycle"""
|
||||
logger.info("Starting GA evolution...")
|
||||
|
||||
# Prepare candle data for fitness evaluation
|
||||
candles_data = {}
|
||||
symbols = self._all_symbols()[:5]
|
||||
|
||||
for symbol in symbols:
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '5m',
|
||||
start=datetime.utcnow() - timedelta(days=14)
|
||||
)
|
||||
if df is None or len(df) < 60:
|
||||
df = self.candle_cache.get_cached(
|
||||
symbol, '1h',
|
||||
start=datetime.utcnow() - timedelta(days=14)
|
||||
)
|
||||
if df is not None and len(df) > 60:
|
||||
featured = self.feature_engine.compute(df)
|
||||
if featured is not None and len(featured) > 50:
|
||||
candles_data[symbol] = featured
|
||||
|
||||
if not candles_data:
|
||||
logger.warning("No data available for GA evolution")
|
||||
return
|
||||
|
||||
# Strategy function factory
|
||||
def strategy_fn_factory(genome):
|
||||
return genome_to_strategy(genome)
|
||||
|
||||
# Run evolution
|
||||
num_gens = self.config['ga'].get('generations_per_cycle', 10)
|
||||
best = self.ga_evolver.run_evolution_cycle(
|
||||
strategy_fn_factory, candles_data, num_generations=num_gens
|
||||
)
|
||||
|
||||
if best:
|
||||
logger.info(f"GA evolution complete: Gen {self.ga_evolver.generation}, "
|
||||
f"best fitness={best.fitness:.4f}")
|
||||
self.krystie.log_ga_milestone(self.ga_evolver.generation, best.fitness)
|
||||
|
||||
def _should_send_daily_report(self, now: datetime) -> bool:
|
||||
"""Check if it's time to send the daily report (once per day after 21:00 UTC)"""
|
||||
report_hour = self.config.get('telegram', {}).get('daily_report_hour', 21)
|
||||
if now.hour >= report_hour and (now - self.last_daily_report).total_seconds() > 20 * 3600:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _send_daily_telegram_report(self):
|
||||
"""Gather data and send the daily Telegram report"""
|
||||
try:
|
||||
alpaca_portfolio = self.broker.get_portfolio()
|
||||
positions = self.broker.get_positions()
|
||||
|
||||
# Combine Alpaca + OANDA portfolios (same logic as dashboard)
|
||||
if self.oanda_broker:
|
||||
try:
|
||||
oanda_portfolio = self.oanda_broker.get_portfolio()
|
||||
oanda_positions = self.oanda_broker.get_positions()
|
||||
positions.extend(oanda_positions)
|
||||
|
||||
combined_equity = alpaca_portfolio['equity'] + oanda_portfolio['equity']
|
||||
combined_day_pnl = alpaca_portfolio['day_pnl'] + oanda_portfolio['day_pnl']
|
||||
prev_equity = combined_equity - combined_day_pnl
|
||||
|
||||
portfolio = {
|
||||
'equity': combined_equity,
|
||||
'cash': alpaca_portfolio['cash'] + oanda_portfolio['cash'],
|
||||
'buying_power': alpaca_portfolio['buying_power'] + oanda_portfolio['buying_power'],
|
||||
'portfolio_value': alpaca_portfolio['portfolio_value'] + oanda_portfolio['portfolio_value'],
|
||||
'long_market_value': alpaca_portfolio['long_market_value'] + oanda_portfolio['long_market_value'],
|
||||
'day_pnl': combined_day_pnl,
|
||||
'day_pnl_pct': (combined_day_pnl / prev_equity * 100) if prev_equity > 0 else 0,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.warning(f"OANDA portfolio fetch failed, using Alpaca only: {e}")
|
||||
portfolio = alpaca_portfolio
|
||||
else:
|
||||
portfolio = alpaca_portfolio
|
||||
|
||||
# Get today's trades from DB
|
||||
today_start = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
today_trades = self.store.get_trades(start=today_start, limit=50)
|
||||
|
||||
# Learning stats
|
||||
rl_stats = self.rl_agent.get_stats()
|
||||
best_genome = self.ga_evolver.get_best_genome()
|
||||
learning_stats = {
|
||||
'rl': rl_stats,
|
||||
'ga': {
|
||||
'generation': self.ga_evolver.generation,
|
||||
'best_fitness': best_genome.fitness if best_genome else 0,
|
||||
},
|
||||
}
|
||||
|
||||
self.telegram.send_daily_report(
|
||||
portfolio, positions, today_trades, learning_stats,
|
||||
self.config['trading']
|
||||
)
|
||||
self.krystie.log_daily_report(
|
||||
equity=portfolio.get('equity', 0),
|
||||
day_pnl=portfolio.get('day_pnl', 0),
|
||||
trades_count=len(today_trades),
|
||||
)
|
||||
logger.info("Daily Telegram report sent")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send daily Telegram report: {e}")
|
||||
|
||||
def _print_dashboard(self):
|
||||
"""Print live console dashboard"""
|
||||
try:
|
||||
# Get Alpaca portfolio
|
||||
alpaca_portfolio = self.broker.get_portfolio()
|
||||
alpaca_positions = self.broker.get_positions()
|
||||
|
||||
# Get OANDA portfolio if available
|
||||
if self.oanda_broker:
|
||||
oanda_portfolio = self.oanda_broker.get_portfolio()
|
||||
oanda_positions = self.oanda_broker.get_positions()
|
||||
|
||||
# Combine portfolios
|
||||
combined_equity = alpaca_portfolio['equity'] + oanda_portfolio['equity']
|
||||
combined_day_pnl = alpaca_portfolio['day_pnl'] + oanda_portfolio['day_pnl']
|
||||
prev_equity = combined_equity - combined_day_pnl
|
||||
|
||||
portfolio = {
|
||||
'equity': combined_equity,
|
||||
'cash': alpaca_portfolio['cash'] + oanda_portfolio['cash'],
|
||||
'buying_power': alpaca_portfolio['buying_power'] + oanda_portfolio['buying_power'],
|
||||
'portfolio_value': alpaca_portfolio['portfolio_value'] + oanda_portfolio['portfolio_value'],
|
||||
'long_market_value': alpaca_portfolio['long_market_value'] + oanda_portfolio['long_market_value'],
|
||||
'day_pnl': combined_day_pnl,
|
||||
'day_pnl_pct': (combined_day_pnl / prev_equity * 100) if prev_equity > 0 else 0,
|
||||
}
|
||||
# Combine positions
|
||||
positions = alpaca_positions + oanda_positions
|
||||
else:
|
||||
portfolio = alpaca_portfolio
|
||||
positions = alpaca_positions
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Dashboard error: {e}")
|
||||
return
|
||||
|
||||
equity = portfolio['equity']
|
||||
target = self.config['trading']['target_capital']
|
||||
|
||||
# Calculate actual initial capital (both brokers start with $100k each in paper trading)
|
||||
if self.oanda_broker:
|
||||
initial = 200000 # $100k Alpaca + $100k OANDA
|
||||
else:
|
||||
initial = 100000 # $100k Alpaca only
|
||||
|
||||
progress = (equity / target) * 100 if target > 0 else 0
|
||||
total_pnl = equity - initial
|
||||
total_pnl_pct = (total_pnl / initial) * 100 if initial > 0 else 0
|
||||
|
||||
uptime = datetime.utcnow() - self.start_time if self.start_time else timedelta()
|
||||
hours = int(uptime.total_seconds() // 3600)
|
||||
minutes = int((uptime.total_seconds() % 3600) // 60)
|
||||
|
||||
best_genome = self.ga_evolver.get_best_genome()
|
||||
gen = self.ga_evolver.generation
|
||||
best_fit = best_genome.fitness if best_genome else 0
|
||||
|
||||
rl_stats = self.rl_agent.get_stats()
|
||||
safety_status = self.safety.get_status()
|
||||
|
||||
stock_status = "OPEN" if self.broker.is_market_open() else "CLOSED"
|
||||
forex_status = ""
|
||||
if self.oanda_broker:
|
||||
forex_status = " | FX: " + ("OPEN" if self.oanda_broker.is_market_open() else "CLOSED")
|
||||
|
||||
# Build dashboard
|
||||
lines = []
|
||||
lines.append("")
|
||||
lines.append("=" * 64)
|
||||
lines.append(f" BIGGFISH AUTONOMOUS TRADER | {hours}h {minutes}m | "
|
||||
f"Gen {gen} | Stocks: {stock_status}{forex_status}")
|
||||
lines.append("=" * 64)
|
||||
|
||||
# Progress bar
|
||||
bar_width = 30
|
||||
filled = int(bar_width * min(progress, 100) / 100)
|
||||
bar = "#" * filled + "-" * (bar_width - filled)
|
||||
lines.append(f" Portfolio: ${equity:.2f} / ${target} [{bar}] {progress:.1f}%")
|
||||
lines.append(f" Day P&L: ${portfolio['day_pnl']:+.2f} ({portfolio['day_pnl_pct']:+.1f}%)")
|
||||
lines.append(f" Total P&L: ${total_pnl:+.2f} ({total_pnl_pct:+.1f}%)")
|
||||
|
||||
# Positions
|
||||
lines.append("-" * 64)
|
||||
if positions:
|
||||
lines.append(f" Active Positions ({len(positions)}):")
|
||||
for p in positions[:5]:
|
||||
pl_str = f"${p['unrealized_pl']:+.2f} ({p['unrealized_plpc']:+.1f}%)"
|
||||
lines.append(f" {p['symbol']:6s} {p['qty']:.0f} @ ${p['avg_entry_price']:.2f}"
|
||||
f" -> ${p['current_price']:.2f} {pl_str}")
|
||||
else:
|
||||
lines.append(" No active positions")
|
||||
|
||||
# Learning status
|
||||
lines.append("-" * 64)
|
||||
lines.append(" Learning Status:")
|
||||
lines.append(f" RL Agent: epsilon={rl_stats['epsilon']:.3f} | "
|
||||
f"loss={rl_stats['avg_loss']:.6f} | "
|
||||
f"{rl_stats['memory_size']:,} experiences")
|
||||
lines.append(f" GA: gen {gen} | best fitness={best_fit:.4f}")
|
||||
|
||||
# Recent trades
|
||||
if self.recent_trades:
|
||||
lines.append("-" * 64)
|
||||
lines.append(" Recent Trades:")
|
||||
for t in self.recent_trades[-5:]:
|
||||
side = t.get('side', '?').upper()
|
||||
symbol = t.get('symbol', '?')
|
||||
pnl = t.get('pnl')
|
||||
if pnl is not None:
|
||||
pnl_str = f" P&L: ${pnl:+.2f}"
|
||||
else:
|
||||
pnl_str = ""
|
||||
price = t.get('entry_price') or t.get('exit_price', 0)
|
||||
lines.append(f" {side:4s} {symbol:6s} "
|
||||
f"{t.get('amount', 0):.1f} @ ${price:.2f}{pnl_str}")
|
||||
|
||||
# Safety
|
||||
if not safety_status['trading_allowed']:
|
||||
lines.append("-" * 64)
|
||||
lines.append(f" !! TRADING HALTED: {safety_status['halt_reason']}")
|
||||
|
||||
# Next events
|
||||
lines.append("-" * 64)
|
||||
now = datetime.utcnow()
|
||||
bt_next = max(0, self.config['backtest']['interval_seconds'] -
|
||||
(now - self.last_backtest).total_seconds())
|
||||
rl_next = max(0, self.config['rl']['train_interval_hours'] * 3600 -
|
||||
(now - self.last_rl_train).total_seconds())
|
||||
ga_next = max(0, self.config['ga']['evolution_interval_hours'] * 3600 -
|
||||
(now - self.last_ga_evolve).total_seconds())
|
||||
|
||||
lines.append(f" Next: backtest {bt_next/60:.0f}m | "
|
||||
f"RL train {rl_next/60:.0f}m | "
|
||||
f"GA evolve {ga_next/3600:.1f}h")
|
||||
lines.append("=" * 64)
|
||||
|
||||
print("\n".join(lines))
|
||||
|
||||
# Update Krystie status file
|
||||
try:
|
||||
markets = {"stocks": stock_status}
|
||||
if self.oanda_broker:
|
||||
markets["forex"] = "OPEN" if self.oanda_broker.is_market_open() else "CLOSED"
|
||||
|
||||
learning_stats = {
|
||||
'rl': rl_stats,
|
||||
'ga': {'generation': gen, 'best_fitness': best_fit},
|
||||
}
|
||||
|
||||
today_start = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
today_trades = self.store.get_trades(start=today_start, limit=50)
|
||||
|
||||
self.krystie.update_status(
|
||||
portfolio=portfolio,
|
||||
positions=positions,
|
||||
learning_stats=learning_stats,
|
||||
markets=markets,
|
||||
config=self.config['trading'],
|
||||
uptime_seconds=uptime.total_seconds(),
|
||||
today_trades=today_trades,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"Krystie status update error: {e}")
|
||||
|
||||
def _load_state(self):
|
||||
"""Load saved state from database"""
|
||||
# Load RL model
|
||||
if self.rl_agent.load(self.store):
|
||||
logger.info("Loaded RL model from checkpoint")
|
||||
|
||||
# Load timing state
|
||||
last_bt = self.store.load_state('last_backtest')
|
||||
if last_bt:
|
||||
self.last_backtest = datetime.fromisoformat(last_bt)
|
||||
|
||||
last_rl = self.store.load_state('last_rl_train')
|
||||
if last_rl:
|
||||
self.last_rl_train = datetime.fromisoformat(last_rl)
|
||||
|
||||
last_ga = self.store.load_state('last_ga_evolve')
|
||||
if last_ga:
|
||||
self.last_ga_evolve = datetime.fromisoformat(last_ga)
|
||||
|
||||
last_report = self.store.load_state('last_daily_report')
|
||||
if last_report:
|
||||
self.last_daily_report = datetime.fromisoformat(last_report)
|
||||
|
||||
logger.info("State loaded from database")
|
||||
|
||||
def _save_state(self):
|
||||
"""Save state to database for recovery"""
|
||||
self.rl_agent.save(self.store)
|
||||
self.store.save_state('last_backtest', self.last_backtest.isoformat())
|
||||
self.store.save_state('last_rl_train', self.last_rl_train.isoformat())
|
||||
self.store.save_state('last_ga_evolve', self.last_ga_evolve.isoformat())
|
||||
self.store.save_state('last_daily_report', self.last_daily_report.isoformat())
|
||||
self.store.save_state('last_shutdown', datetime.utcnow().isoformat())
|
||||
logger.info("State saved to database")
|
||||
|
||||
def _signal_handler(self, signum, frame):
|
||||
"""Handle SIGINT for graceful shutdown"""
|
||||
self._shutdown()
|
||||
|
||||
def _shutdown(self):
|
||||
"""Graceful shutdown"""
|
||||
logger.info("Shutting down BIGGFISH...")
|
||||
self.running = False
|
||||
self.krystie.log_shutdown()
|
||||
self._save_state()
|
||||
self.store.close()
|
||||
logger.info("Shutdown complete. State saved. Resume anytime.")
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description="BIGGFISH Autonomous Trader")
|
||||
parser.add_argument('--config', default='config/auto_config.json',
|
||||
help='Path to configuration file')
|
||||
args = parser.parse_args()
|
||||
|
||||
config_path = Path(args.config)
|
||||
if not config_path.exists():
|
||||
print(f"Config file not found: {config_path}")
|
||||
print("Copy config/auto_config.json and fill in your Alpaca API keys")
|
||||
sys.exit(1)
|
||||
|
||||
bot = BiggFishAuto(str(config_path))
|
||||
bot.start()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+141
-17
@@ -3,11 +3,17 @@ Genetic Algorithm for Strategy Parameter Optimization
|
||||
Evolves a population of StrategyGenome instances to find profitable trading parameters.
|
||||
|
||||
Optimized: uses vectorized backtesting and parallel genome evaluation.
|
||||
Enhanced with:
|
||||
- Recent drawdown protection (last 20% of trades weighted more heavily)
|
||||
- Correlation penalty (diversification bonus)
|
||||
- Calmar/Sortino-enhanced Sharpe ratio
|
||||
- Market regime detection (trending vs ranging) for adaptive fitness
|
||||
"""
|
||||
|
||||
import random
|
||||
import math
|
||||
import json
|
||||
import numpy as np
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
@@ -137,34 +143,109 @@ def _evaluate_genome_worker(genome_dict: Dict, candles_dict: Dict[str, Dict],
|
||||
|
||||
m = result.metrics
|
||||
sharpe = max(m.get('sharpe_ratio', 0), 0)
|
||||
sortino = max(m.get('sortino_ratio', 0), 0)
|
||||
max_dd = m.get('max_drawdown', 0)
|
||||
max_dd_dur = m.get('max_drawdown_duration', 0)
|
||||
total_trades = m.get('total_trades', 0)
|
||||
win_rate = m.get('win_rate', 0) / 100.0 # 0-1
|
||||
total_return = m.get('total_return', 0) / 100.0 # Convert % to decimal
|
||||
|
||||
# CRITICAL FIX: Profit/return MUST be the primary fitness component
|
||||
# Without this, GA evolves "good metrics" that lose money!
|
||||
|
||||
# Return component (most important): exponential reward for profit, penalty for loss
|
||||
trades = result.trades
|
||||
|
||||
# ─── 1. RECENT DRAWDOWN PROTECTION ─────────────────────────────────────
|
||||
# Weight last 20% of trades 3x more than early trades (recent performance matters)
|
||||
recent_weight = 0.0
|
||||
if trades and len(trades) >= 5:
|
||||
n = len(trades)
|
||||
cutoff = int(n * 0.8)
|
||||
recent_trades = trades[cutoff:]
|
||||
early_trades = trades[:cutoff]
|
||||
|
||||
early_pnl = sum(t.get('pnl', 0) for t in early_trades)
|
||||
recent_pnl = sum(t.get('pnl', 0) for t in recent_trades)
|
||||
|
||||
# If recent performance is negative while early was positive → severe penalty
|
||||
if early_pnl > 0 and recent_pnl < 0:
|
||||
# Struggling in recent market conditions
|
||||
recent_weight = -abs(recent_pnl) * 5.0
|
||||
elif recent_pnl < 0:
|
||||
recent_weight = -abs(recent_pnl) * 3.0
|
||||
elif recent_pnl > 0:
|
||||
recent_weight = recent_pnl * 2.0 # Bonus for profitable recent trades
|
||||
elif trades:
|
||||
recent_pnl = sum(t.get('pnl', 0) for t in trades)
|
||||
recent_weight = recent_pnl * 1.5
|
||||
|
||||
# ─── 2. CORRELATION / CONCENTRATION PENALTY ────────────────────────────
|
||||
# Count short vs long trades; penalize if ALL trades are same direction
|
||||
# (vulnerability to market reversal)
|
||||
short_trades = [t for t in trades if t.get('side') == 'short']
|
||||
long_trades = [t for t in trades if t.get('side') == 'long']
|
||||
n_st = len(short_trades)
|
||||
n_lt = len(long_trades)
|
||||
n_total = n_st + n_lt
|
||||
|
||||
direction_penalty = 0.0
|
||||
if n_total >= 5:
|
||||
short_frac = n_st / n_total
|
||||
# Penalty for >80% concentration in shorts OR longs
|
||||
if short_frac > 0.80:
|
||||
direction_penalty = -(1 - short_frac) * 2.0 # Too many shorts
|
||||
elif short_frac < 0.20:
|
||||
direction_penalty = -(short_frac) * 2.0 # Too many longs
|
||||
|
||||
# ─── 3. ENHANCED SHARPE (Calmar + Sortino blend) ───────────────────────
|
||||
# Calmar = return / max_dd (higher = better risk-adjusted)
|
||||
calmar = (total_return / (max_dd / 100)) if max_dd > 0.5 else 0.0
|
||||
|
||||
# Risk-adjusted score: blend Sharpe, Sortino, Calmar
|
||||
risk_adj_score = (sharpe * 0.4 + sortino * 0.3 + min(max(calmar, 0), 10) * 0.3)
|
||||
|
||||
# ─── 4. MARKET REGIME DETECTION ─────────────────────────────────────────
|
||||
# Detect if market is trending or ranging from recent returns
|
||||
regime_bonus = 0.0
|
||||
if trades and len(trades) >= 3:
|
||||
trade_returns = [t.get('pnl_pct', 0) for t in trades]
|
||||
# High variance = trending; low variance = ranging
|
||||
mean_ret = np.mean(trade_returns)
|
||||
std_ret = max(np.std(trade_returns), 0.01)
|
||||
# Consistency ratio: are returns stable?
|
||||
consistency = mean_ret / std_ret if std_ret > 0 else 0
|
||||
# Bonus for maintaining positive returns in high-variance (trending) markets
|
||||
if abs(consistency) > 1.5 and mean_ret > 0:
|
||||
regime_bonus = mean_ret * 0.5
|
||||
|
||||
# ─── PRIMARY FITNESS: Total Return ─────────────────────────────────────
|
||||
if total_return > 0:
|
||||
return_score = 1.0 + (total_return * 10.0) # Reward profit heavily
|
||||
return_score = 1.0 + (total_return * 10.0)
|
||||
else:
|
||||
return_score = max(0.01, 1.0 + (total_return * 20.0)) # Penalize losses even harder
|
||||
|
||||
return_score = max(0.01, 1.0 + (total_return * 20.0))
|
||||
|
||||
dd_penalty = max(1 - max_dd / 100, 0)
|
||||
# Scalping: reward higher trade frequency (but less than before)
|
||||
# Scalping: reward higher trade frequency
|
||||
trade_bonus = 1.0 + math.log1p(max(total_trades, 0)) * 0.1
|
||||
# Bonus for win rate > 50%
|
||||
wr_bonus = 1.0 + max(0, win_rate - 0.5) * 0.3
|
||||
|
||||
|
||||
# Sharpe bonus (risk-adjusted return quality)
|
||||
sharpe_bonus = 1.0 + (sharpe * 0.2)
|
||||
sharpe_bonus = 1.0 + (risk_adj_score * 0.2)
|
||||
|
||||
if total_trades < 5:
|
||||
trade_bonus *= 0.3 # Scalping needs more trades
|
||||
|
||||
# NEW FORMULA: Profit is the PRIMARY driver, everything else modulates it
|
||||
score = return_score * sharpe_bonus * dd_penalty * trade_bonus * wr_bonus
|
||||
# ─── COMBINED FITNESS SCORE ─────────────────────────────────────────────
|
||||
# Profit is the PRIMARY driver; recent protection, correlation, regime
|
||||
# and risk-adjusted metrics MODULATE it
|
||||
score = (return_score
|
||||
* sharpe_bonus
|
||||
* dd_penalty
|
||||
* trade_bonus
|
||||
* wr_bonus
|
||||
+ recent_weight
|
||||
+ direction_penalty
|
||||
+ regime_bonus)
|
||||
|
||||
# Floor: very negative recent form can't sink below tiny positive floor
|
||||
score = max(score, 0.001)
|
||||
fitness_scores.append(score)
|
||||
|
||||
if not fitness_scores:
|
||||
@@ -247,25 +328,68 @@ class GeneticEvolver:
|
||||
|
||||
m = result.metrics
|
||||
sharpe = max(m.get('sharpe_ratio', 0), 0)
|
||||
sortino = max(m.get('sortino_ratio', 0), 0)
|
||||
max_dd = m.get('max_drawdown', 0)
|
||||
total_trades = m.get('total_trades', 0)
|
||||
win_rate = m.get('win_rate', 0) / 100.0
|
||||
total_return = m.get('total_return', 0) / 100.0 # Convert % to decimal
|
||||
trades = result.trades
|
||||
|
||||
# ─── Recent drawdown protection ───────────────────────────────────
|
||||
recent_weight = 0.0
|
||||
if trades and len(trades) >= 5:
|
||||
n = len(trades)
|
||||
cutoff = int(n * 0.8)
|
||||
recent_trades = trades[cutoff:]
|
||||
recent_pnl = sum(t.get('pnl', 0) for t in recent_trades)
|
||||
early_pnl = sum(t.get('pnl', 0) for t in trades[:cutoff])
|
||||
if early_pnl > 0 and recent_pnl < 0:
|
||||
recent_weight = -abs(recent_pnl) * 5.0
|
||||
elif recent_pnl < 0:
|
||||
recent_weight = -abs(recent_pnl) * 3.0
|
||||
elif recent_pnl > 0:
|
||||
recent_weight = recent_pnl * 2.0
|
||||
elif trades:
|
||||
recent_pnl = sum(t.get('pnl', 0) for t in trades)
|
||||
recent_weight = recent_pnl * 1.5
|
||||
|
||||
# ─── Correlation / direction concentration ─────────────────────────
|
||||
direction_penalty = 0.0
|
||||
short_trades = [t for t in trades if t.get('side') == 'short']
|
||||
long_trades = [t for t in trades if t.get('side') == 'long']
|
||||
n_total = len(short_trades) + len(long_trades)
|
||||
if n_total >= 5:
|
||||
short_frac = len(short_trades) / n_total
|
||||
if short_frac > 0.80:
|
||||
direction_penalty = -(1 - short_frac) * 2.0
|
||||
elif short_frac < 0.20:
|
||||
direction_penalty = -(short_frac) * 2.0
|
||||
|
||||
# ─── Enhanced risk-adjusted score ──────────────────────────────────
|
||||
calmar = (total_return / (max_dd / 100)) if max_dd > 0.5 else 0.0
|
||||
risk_adj_score = (sharpe * 0.4 + sortino * 0.3 + min(max(calmar, 0), 10) * 0.3)
|
||||
|
||||
# Fitness = profit-weighted Sharpe with safety constraints
|
||||
dd_penalty = max(1 - max_dd / 100, 0)
|
||||
trade_bonus = math.sqrt(max(total_trades, 0))
|
||||
wr_bonus = 1.0 + max(0, win_rate - 0.5) * 0.5
|
||||
|
||||
|
||||
# Weight actual profit heavily (10x multiplier)
|
||||
profit_score = max(0, total_return) * 10
|
||||
|
||||
|
||||
# Penalize strategies with few trades
|
||||
if total_trades < 5:
|
||||
trade_bonus *= 0.3
|
||||
|
||||
# Combined score: profit is primary, Sharpe/WR/DD are modifiers
|
||||
score = profit_score * (1 + sharpe) * dd_penalty * trade_bonus * wr_bonus
|
||||
# Combined score: profit is primary, everything else modulates
|
||||
score = (profit_score
|
||||
* (1 + risk_adj_score * 0.2)
|
||||
* dd_penalty
|
||||
* trade_bonus
|
||||
* wr_bonus
|
||||
+ recent_weight
|
||||
+ direction_penalty)
|
||||
score = max(score, 0.001)
|
||||
fitness_scores.append(score)
|
||||
|
||||
if not fitness_scores:
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
"""
|
||||
Genetic Algorithm for Strategy Parameter Optimization
|
||||
Evolves a population of StrategyGenome instances to find profitable trading parameters.
|
||||
|
||||
Optimized: uses vectorized backtesting and parallel genome evaluation.
|
||||
"""
|
||||
|
||||
import random
|
||||
import math
|
||||
import json
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from loguru import logger
|
||||
|
||||
|
||||
# Gene ranges for scalping: (min, max, is_int)
|
||||
# Shorter periods, tighter SL/TP, faster hold times
|
||||
GENE_RANGES = {
|
||||
'fast_ma_period': (3, 20, True),
|
||||
'slow_ma_period': (10, 60, True),
|
||||
'rsi_period': (5, 21, True),
|
||||
'rsi_overbought': (60, 80, False),
|
||||
'rsi_oversold': (20, 40, False),
|
||||
'bb_period': (8, 25, True),
|
||||
'bb_std': (1.5, 2.5, False),
|
||||
'atr_period': (5, 14, True),
|
||||
'macd_fast': (5, 12, True),
|
||||
'macd_slow': (12, 26, True),
|
||||
'macd_signal': (5, 9, True),
|
||||
'volume_surge_threshold': (1.1, 2.5, False),
|
||||
'stop_loss_atr_mult': (0.5, 2.5, False),
|
||||
'take_profit_atr_mult': (0.8, 3.0, False),
|
||||
'max_position_pct': (0.05, 0.15, False),
|
||||
'min_hold_candles': (1, 6, True),
|
||||
'max_hold_candles': (3, 36, True),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class StrategyGenome:
|
||||
"""A genome encoding all tunable strategy parameters"""
|
||||
# Indicator periods (scalping-tuned defaults)
|
||||
fast_ma_period: int = 8
|
||||
slow_ma_period: int = 21
|
||||
rsi_period: int = 9
|
||||
rsi_overbought: float = 70.0
|
||||
rsi_oversold: float = 30.0
|
||||
bb_period: int = 15
|
||||
bb_std: float = 2.0
|
||||
atr_period: int = 10
|
||||
macd_fast: int = 8
|
||||
macd_slow: int = 17
|
||||
macd_signal: int = 7
|
||||
|
||||
# Entry thresholds
|
||||
volume_surge_threshold: float = 1.3
|
||||
|
||||
# Risk management (tighter for scalping)
|
||||
stop_loss_atr_mult: float = 1.2
|
||||
take_profit_atr_mult: float = 1.8
|
||||
max_position_pct: float = 0.10
|
||||
|
||||
# Timing (short holds for scalping)
|
||||
min_hold_candles: int = 1
|
||||
max_hold_candles: int = 12
|
||||
|
||||
# Fitness (set after evaluation)
|
||||
fitness: float = 0.0
|
||||
generation: int = 0
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict) -> 'StrategyGenome':
|
||||
valid_fields = {f.name for f in cls.__dataclass_fields__.values()}
|
||||
filtered = {k: v for k, v in d.items() if k in valid_fields}
|
||||
return cls(**filtered)
|
||||
|
||||
@classmethod
|
||||
def random(cls, generation: int = 0) -> 'StrategyGenome':
|
||||
"""Create a random genome within valid parameter ranges"""
|
||||
genes = {}
|
||||
for gene_name, (lo, hi, is_int) in GENE_RANGES.items():
|
||||
if is_int:
|
||||
genes[gene_name] = random.randint(int(lo), int(hi))
|
||||
else:
|
||||
genes[gene_name] = round(random.uniform(lo, hi), 4)
|
||||
|
||||
# Constraint: slow_ma > fast_ma
|
||||
if genes['slow_ma_period'] <= genes['fast_ma_period']:
|
||||
genes['slow_ma_period'] = genes['fast_ma_period'] + random.randint(10, 50)
|
||||
|
||||
# Constraint: macd_slow > macd_fast
|
||||
if genes['macd_slow'] <= genes['macd_fast']:
|
||||
genes['macd_slow'] = genes['macd_fast'] + random.randint(8, 16)
|
||||
|
||||
# Constraint: take_profit > stop_loss
|
||||
if genes['take_profit_atr_mult'] <= genes['stop_loss_atr_mult']:
|
||||
genes['take_profit_atr_mult'] = genes['stop_loss_atr_mult'] + random.uniform(0.5, 2.0)
|
||||
|
||||
# Constraint: max_hold > min_hold
|
||||
if genes['max_hold_candles'] <= genes['min_hold_candles']:
|
||||
genes['max_hold_candles'] = genes['min_hold_candles'] + random.randint(10, 50)
|
||||
|
||||
genes['generation'] = generation
|
||||
return cls(**genes)
|
||||
|
||||
|
||||
def _evaluate_genome_worker(genome_dict: Dict, candles_dict: Dict[str, Dict],
|
||||
initial_capital: float, commission_rate: float) -> float:
|
||||
"""
|
||||
Worker function for parallel genome evaluation.
|
||||
Runs in a separate process, so must be a top-level function.
|
||||
Uses vectorized signal generation + fast backtest.
|
||||
"""
|
||||
from strategies.auto_strategy import genome_to_signals
|
||||
from backtest.engine import BacktestEngine
|
||||
import pandas as pd
|
||||
|
||||
genome = StrategyGenome.from_dict(genome_dict)
|
||||
engine = BacktestEngine(initial_capital=initial_capital, commission_rate=commission_rate)
|
||||
|
||||
fitness_scores = []
|
||||
|
||||
for symbol, candle_data in candles_dict.items():
|
||||
df = pd.DataFrame(candle_data)
|
||||
if len(df) < 60:
|
||||
continue
|
||||
|
||||
try:
|
||||
signals = genome_to_signals(genome, df)
|
||||
result = engine.run_fast(signals, df, symbol=symbol)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
m = result.metrics
|
||||
sharpe = max(m.get('sharpe_ratio', 0), 0)
|
||||
max_dd = m.get('max_drawdown', 0)
|
||||
total_trades = m.get('total_trades', 0)
|
||||
win_rate = m.get('win_rate', 0) / 100.0 # 0-1
|
||||
total_return = m.get('total_return', 0) / 100.0 # Convert % to decimal
|
||||
|
||||
# CRITICAL FIX: Profit/return MUST be the primary fitness component
|
||||
# Without this, GA evolves "good metrics" that lose money!
|
||||
|
||||
# Return component (most important): exponential reward for profit, penalty for loss
|
||||
if total_return > 0:
|
||||
return_score = 1.0 + (total_return * 10.0) # Reward profit heavily
|
||||
else:
|
||||
return_score = max(0.01, 1.0 + (total_return * 20.0)) # Penalize losses even harder
|
||||
|
||||
dd_penalty = max(1 - max_dd / 100, 0)
|
||||
# Scalping: reward higher trade frequency (but less than before)
|
||||
trade_bonus = 1.0 + math.log1p(max(total_trades, 0)) * 0.1
|
||||
# Bonus for win rate > 50%
|
||||
wr_bonus = 1.0 + max(0, win_rate - 0.5) * 0.3
|
||||
|
||||
# Sharpe bonus (risk-adjusted return quality)
|
||||
sharpe_bonus = 1.0 + (sharpe * 0.2)
|
||||
|
||||
if total_trades < 5:
|
||||
trade_bonus *= 0.3 # Scalping needs more trades
|
||||
|
||||
# NEW FORMULA: Profit is the PRIMARY driver, everything else modulates it
|
||||
score = return_score * sharpe_bonus * dd_penalty * trade_bonus * wr_bonus
|
||||
fitness_scores.append(score)
|
||||
|
||||
if not fitness_scores:
|
||||
return 0.0
|
||||
|
||||
return sum(fitness_scores) / len(fitness_scores)
|
||||
|
||||
|
||||
class GeneticEvolver:
|
||||
"""
|
||||
Genetic algorithm for optimizing strategy parameters.
|
||||
Evolves a population of StrategyGenome instances.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict, backtest_engine, store=None):
|
||||
self.population_size = config.get('population_size', 50)
|
||||
self.elite_count = config.get('elite_count', 5)
|
||||
self.mutation_rate = config.get('mutation_rate', 0.15)
|
||||
self.mutation_strength = config.get('mutation_strength', 0.2)
|
||||
self.crossover_rate = config.get('crossover_rate', 0.7)
|
||||
self.tournament_size = config.get('tournament_size', 5)
|
||||
self.parallel_workers = config.get('parallel_workers', 4)
|
||||
|
||||
self.backtest_engine = backtest_engine
|
||||
self.store = store
|
||||
self.population: List[StrategyGenome] = []
|
||||
self.generation = 0
|
||||
self.best_ever: Optional[StrategyGenome] = None
|
||||
|
||||
def initialize_population(self):
|
||||
"""
|
||||
Create initial population.
|
||||
If evolution history exists in DB, load the latest generation.
|
||||
Otherwise, create random genomes.
|
||||
"""
|
||||
if self.store:
|
||||
latest = self.store.get_latest_generation()
|
||||
if latest:
|
||||
self.generation = latest['generation']
|
||||
self.population = [
|
||||
StrategyGenome.from_dict(g) for g in latest['population']
|
||||
]
|
||||
if self.population:
|
||||
self.best_ever = max(self.population, key=lambda g: g.fitness)
|
||||
logger.info(f"Loaded GA population from generation {self.generation} "
|
||||
f"({len(self.population)} genomes)")
|
||||
return
|
||||
|
||||
# Create random population
|
||||
self.population = [
|
||||
StrategyGenome.random(generation=0)
|
||||
for _ in range(self.population_size)
|
||||
]
|
||||
self.generation = 0
|
||||
logger.info(f"Created random population of {self.population_size} genomes")
|
||||
|
||||
def evaluate_fitness(self, genome: StrategyGenome, strategy_fn_factory,
|
||||
candles_data: Dict[str, 'pd.DataFrame']) -> float:
|
||||
"""
|
||||
Evaluate a genome using vectorized fast backtest.
|
||||
Falls back to callback-based if signals not available.
|
||||
"""
|
||||
from strategies.auto_strategy import genome_to_signals
|
||||
|
||||
fitness_scores = []
|
||||
|
||||
for symbol, df in candles_data.items():
|
||||
if df is None or len(df) < 60:
|
||||
continue
|
||||
|
||||
try:
|
||||
signals = genome_to_signals(genome, df)
|
||||
result = self.backtest_engine.run_fast(signals, df, symbol=symbol)
|
||||
except Exception:
|
||||
# Fallback to callback-based
|
||||
strategy_fn = strategy_fn_factory(genome)
|
||||
result = self.backtest_engine.run(
|
||||
strategy_fn, df, params=genome.to_dict(), symbol=symbol
|
||||
)
|
||||
|
||||
m = result.metrics
|
||||
sharpe = max(m.get('sharpe_ratio', 0), 0)
|
||||
max_dd = m.get('max_drawdown', 0)
|
||||
total_trades = m.get('total_trades', 0)
|
||||
win_rate = m.get('win_rate', 0) / 100.0
|
||||
total_return = m.get('total_return', 0) / 100.0 # Convert % to decimal
|
||||
|
||||
# Fitness = profit-weighted Sharpe with safety constraints
|
||||
dd_penalty = max(1 - max_dd / 100, 0)
|
||||
trade_bonus = math.sqrt(max(total_trades, 0))
|
||||
wr_bonus = 1.0 + max(0, win_rate - 0.5) * 0.5
|
||||
|
||||
# Weight actual profit heavily (10x multiplier)
|
||||
profit_score = max(0, total_return) * 10
|
||||
|
||||
# Penalize strategies with few trades
|
||||
if total_trades < 5:
|
||||
trade_bonus *= 0.3
|
||||
|
||||
# Combined score: profit is primary, Sharpe/WR/DD are modifiers
|
||||
score = profit_score * (1 + sharpe) * dd_penalty * trade_bonus * wr_bonus
|
||||
fitness_scores.append(score)
|
||||
|
||||
if not fitness_scores:
|
||||
return 0.0
|
||||
|
||||
return sum(fitness_scores) / len(fitness_scores)
|
||||
|
||||
def evaluate_population(self, strategy_fn_factory,
|
||||
candles_data: Dict[str, 'pd.DataFrame']):
|
||||
"""Evaluate all genomes - uses parallel workers if available."""
|
||||
unevaluated = [(i, g) for i, g in enumerate(self.population) if g.fitness == 0.0]
|
||||
|
||||
if not unevaluated:
|
||||
return
|
||||
|
||||
# Prepare serializable candle data for parallel workers
|
||||
candles_dict = {}
|
||||
for symbol, df in candles_data.items():
|
||||
if df is not None and len(df) >= 60:
|
||||
candles_dict[symbol] = {
|
||||
'open': df['open'].values.tolist(),
|
||||
'high': df['high'].values.tolist(),
|
||||
'low': df['low'].values.tolist(),
|
||||
'close': df['close'].values.tolist(),
|
||||
'volume': df['volume'].values.tolist(),
|
||||
}
|
||||
|
||||
if not candles_dict:
|
||||
return
|
||||
|
||||
initial_capital = self.backtest_engine.initial_capital
|
||||
commission_rate = self.backtest_engine.commission_rate
|
||||
|
||||
# Try parallel evaluation
|
||||
if self.parallel_workers > 1 and len(unevaluated) > 4:
|
||||
try:
|
||||
self._evaluate_parallel(
|
||||
unevaluated, candles_dict, initial_capital,
|
||||
commission_rate, strategy_fn_factory, candles_data
|
||||
)
|
||||
return
|
||||
except Exception as e:
|
||||
logger.debug(f"Parallel evaluation failed, falling back to sequential: {e}")
|
||||
|
||||
# Sequential fallback (still uses vectorized fast path)
|
||||
for idx, (i, genome) in enumerate(unevaluated):
|
||||
genome.fitness = self.evaluate_fitness(
|
||||
genome, strategy_fn_factory, candles_data
|
||||
)
|
||||
if (idx + 1) % 10 == 0:
|
||||
logger.debug(f"Evaluated {idx+1}/{len(unevaluated)} genomes")
|
||||
|
||||
def _evaluate_parallel(self, unevaluated, candles_dict, initial_capital,
|
||||
commission_rate, strategy_fn_factory, candles_data):
|
||||
"""Evaluate genomes in parallel using ProcessPoolExecutor."""
|
||||
genome_dicts = [(i, g.to_dict()) for i, g in unevaluated]
|
||||
|
||||
with ProcessPoolExecutor(max_workers=self.parallel_workers) as executor:
|
||||
futures = {}
|
||||
for i, gd in genome_dicts:
|
||||
fut = executor.submit(
|
||||
_evaluate_genome_worker, gd, candles_dict,
|
||||
initial_capital, commission_rate
|
||||
)
|
||||
futures[fut] = i
|
||||
|
||||
done_count = 0
|
||||
for future in as_completed(futures):
|
||||
pop_idx = futures[future]
|
||||
try:
|
||||
fitness = future.result(timeout=30)
|
||||
self.population[pop_idx].fitness = fitness
|
||||
except Exception:
|
||||
# Fallback for this genome
|
||||
self.population[pop_idx].fitness = self.evaluate_fitness(
|
||||
self.population[pop_idx], strategy_fn_factory, candles_data
|
||||
)
|
||||
done_count += 1
|
||||
if done_count % 10 == 0:
|
||||
logger.debug(f"Evaluated {done_count}/{len(unevaluated)} genomes (parallel)")
|
||||
|
||||
def select_parent(self) -> StrategyGenome:
|
||||
"""Tournament selection"""
|
||||
tournament = random.sample(
|
||||
self.population,
|
||||
min(self.tournament_size, len(self.population))
|
||||
)
|
||||
return max(tournament, key=lambda g: g.fitness)
|
||||
|
||||
def crossover(self, parent1: StrategyGenome,
|
||||
parent2: StrategyGenome) -> StrategyGenome:
|
||||
"""Uniform crossover: for each gene, randomly pick from parent1 or parent2"""
|
||||
child_genes = {}
|
||||
p1 = parent1.to_dict()
|
||||
p2 = parent2.to_dict()
|
||||
|
||||
for gene_name in GENE_RANGES:
|
||||
child_genes[gene_name] = p1[gene_name] if random.random() < 0.5 else p2[gene_name]
|
||||
|
||||
# Repair constraints
|
||||
if child_genes['slow_ma_period'] <= child_genes['fast_ma_period']:
|
||||
child_genes['slow_ma_period'] = child_genes['fast_ma_period'] + 10
|
||||
|
||||
if child_genes['macd_slow'] <= child_genes['macd_fast']:
|
||||
child_genes['macd_slow'] = child_genes['macd_fast'] + 8
|
||||
|
||||
if child_genes['take_profit_atr_mult'] <= child_genes['stop_loss_atr_mult']:
|
||||
child_genes['take_profit_atr_mult'] = child_genes['stop_loss_atr_mult'] + 0.5
|
||||
|
||||
if child_genes['max_hold_candles'] <= child_genes['min_hold_candles']:
|
||||
child_genes['max_hold_candles'] = child_genes['min_hold_candles'] + 10
|
||||
|
||||
child_genes['fitness'] = 0.0
|
||||
child_genes['generation'] = self.generation + 1
|
||||
return StrategyGenome(**child_genes)
|
||||
|
||||
def mutate(self, genome: StrategyGenome) -> StrategyGenome:
|
||||
"""Gaussian mutation on each gene with probability mutation_rate"""
|
||||
genes = genome.to_dict()
|
||||
|
||||
for gene_name, (lo, hi, is_int) in GENE_RANGES.items():
|
||||
if random.random() < self.mutation_rate:
|
||||
gene_range = hi - lo
|
||||
delta = random.gauss(0, gene_range * self.mutation_strength)
|
||||
|
||||
new_val = genes[gene_name] + delta
|
||||
new_val = max(lo, min(hi, new_val))
|
||||
|
||||
if is_int:
|
||||
new_val = round(new_val)
|
||||
else:
|
||||
new_val = round(new_val, 4)
|
||||
|
||||
genes[gene_name] = new_val
|
||||
|
||||
# Repair constraints after mutation
|
||||
if genes['slow_ma_period'] <= genes['fast_ma_period']:
|
||||
genes['slow_ma_period'] = genes['fast_ma_period'] + 10
|
||||
if genes['macd_slow'] <= genes['macd_fast']:
|
||||
genes['macd_slow'] = genes['macd_fast'] + 8
|
||||
if genes['take_profit_atr_mult'] <= genes['stop_loss_atr_mult']:
|
||||
genes['take_profit_atr_mult'] = genes['stop_loss_atr_mult'] + 0.5
|
||||
if genes['max_hold_candles'] <= genes['min_hold_candles']:
|
||||
genes['max_hold_candles'] = genes['min_hold_candles'] + 10
|
||||
|
||||
genes['fitness'] = 0.0
|
||||
genes['generation'] = self.generation + 1
|
||||
return StrategyGenome.from_dict(genes)
|
||||
|
||||
def evolve_generation(self, strategy_fn_factory,
|
||||
candles_data: Dict[str, 'pd.DataFrame']) -> Dict:
|
||||
"""
|
||||
Run one generation of evolution:
|
||||
1. Evaluate fitness of all genomes
|
||||
2. Sort by fitness
|
||||
3. Keep elite_count best unchanged
|
||||
4. Fill remaining via tournament selection + crossover + mutation
|
||||
5. Save to database
|
||||
"""
|
||||
# Evaluate
|
||||
self.evaluate_population(strategy_fn_factory, candles_data)
|
||||
|
||||
# Sort by fitness
|
||||
self.population.sort(key=lambda g: g.fitness, reverse=True)
|
||||
|
||||
best = self.population[0]
|
||||
avg_fitness = sum(g.fitness for g in self.population) / len(self.population)
|
||||
|
||||
if self.best_ever is None or best.fitness > self.best_ever.fitness:
|
||||
self.best_ever = StrategyGenome.from_dict(best.to_dict())
|
||||
self.best_ever.fitness = best.fitness
|
||||
|
||||
# Elitism: keep top N unchanged
|
||||
new_population = [
|
||||
StrategyGenome.from_dict(g.to_dict())
|
||||
for g in self.population[:self.elite_count]
|
||||
]
|
||||
# Preserve their fitness
|
||||
for i in range(min(self.elite_count, len(self.population))):
|
||||
new_population[i].fitness = self.population[i].fitness
|
||||
|
||||
# Fill the rest
|
||||
while len(new_population) < self.population_size:
|
||||
parent1 = self.select_parent()
|
||||
parent2 = self.select_parent()
|
||||
|
||||
if random.random() < self.crossover_rate:
|
||||
child = self.crossover(parent1, parent2)
|
||||
else:
|
||||
child = StrategyGenome.from_dict(parent1.to_dict())
|
||||
child.fitness = 0.0
|
||||
|
||||
child = self.mutate(child)
|
||||
new_population.append(child)
|
||||
|
||||
self.population = new_population
|
||||
self.generation += 1
|
||||
|
||||
# Save to database
|
||||
if self.store:
|
||||
self.store.record_generation(
|
||||
self.generation,
|
||||
[g.to_dict() for g in self.population],
|
||||
best.fitness,
|
||||
avg_fitness
|
||||
)
|
||||
|
||||
stats = {
|
||||
'generation': self.generation,
|
||||
'best_fitness': round(best.fitness, 4),
|
||||
'avg_fitness': round(avg_fitness, 4),
|
||||
'best_genome': best.to_dict(),
|
||||
}
|
||||
|
||||
logger.info(f"GA Gen {self.generation}: best={best.fitness:.4f} avg={avg_fitness:.4f}")
|
||||
return stats
|
||||
|
||||
def run_evolution_cycle(self, strategy_fn_factory,
|
||||
candles_data: Dict[str, 'pd.DataFrame'],
|
||||
num_generations: int = 10) -> Optional[StrategyGenome]:
|
||||
"""
|
||||
Run multiple generations.
|
||||
Returns the best genome found.
|
||||
"""
|
||||
for _ in range(num_generations):
|
||||
self.evolve_generation(strategy_fn_factory, candles_data)
|
||||
|
||||
return self.get_best_genome()
|
||||
|
||||
def get_best_genome(self) -> Optional[StrategyGenome]:
|
||||
"""Return the highest-fitness genome"""
|
||||
if self.best_ever:
|
||||
return self.best_ever
|
||||
if self.population:
|
||||
return max(self.population, key=lambda g: g.fitness)
|
||||
return None
|
||||
@@ -0,0 +1,328 @@
|
||||
"""
|
||||
Reinforcement Learning Agent for Trading
|
||||
DQN with experience replay and target network, implemented in PyTorch.
|
||||
"""
|
||||
|
||||
import io
|
||||
import random
|
||||
import numpy as np
|
||||
from collections import deque
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from loguru import logger
|
||||
|
||||
try:
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.optim as optim
|
||||
TORCH_AVAILABLE = True
|
||||
except ImportError:
|
||||
TORCH_AVAILABLE = False
|
||||
logger.warning("PyTorch not installed. RL agent will use random actions. "
|
||||
"Install with: pip install torch")
|
||||
|
||||
|
||||
class TradingNetwork:
|
||||
"""Neural network for the RL agent (PyTorch or fallback)"""
|
||||
pass
|
||||
|
||||
|
||||
if TORCH_AVAILABLE:
|
||||
class TradingNetwork(nn.Module):
|
||||
"""MLP with 2 hidden layers for Q-value prediction"""
|
||||
|
||||
def __init__(self, state_dim: int, action_dim: int, hidden_dim: int = 128):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Linear(state_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.1),
|
||||
nn.Linear(hidden_dim, hidden_dim),
|
||||
nn.ReLU(),
|
||||
nn.Dropout(0.1),
|
||||
nn.Linear(hidden_dim, action_dim)
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
class RLAgent:
|
||||
"""
|
||||
DQN-based RL agent for trading decisions.
|
||||
Falls back to random actions if PyTorch is not available.
|
||||
"""
|
||||
|
||||
def __init__(self, state_dim: int, action_dim: int = 7, config: Dict = None):
|
||||
config = config or {}
|
||||
self.config = config # CRITICAL FIX: Save config for memory persistence
|
||||
self.state_dim = state_dim
|
||||
self.action_dim = action_dim
|
||||
|
||||
# Hyperparameters
|
||||
self.gamma = config.get('gamma', 0.99)
|
||||
self.epsilon = config.get('epsilon_start', 1.0)
|
||||
self.epsilon_min = config.get('epsilon_min', 0.05)
|
||||
self.epsilon_decay = config.get('epsilon_decay', 0.9995)
|
||||
self.learning_rate = config.get('learning_rate', 0.0003)
|
||||
self.batch_size = config.get('batch_size', 64)
|
||||
self.memory_size = config.get('memory_size', 50000)
|
||||
self.target_update_freq = config.get('target_update_freq', 100)
|
||||
self.live_epsilon = config.get('live_epsilon', 0.1)
|
||||
|
||||
# Experience replay buffer
|
||||
self.memory = deque(maxlen=self.memory_size)
|
||||
self.steps = 0
|
||||
self.training_losses = []
|
||||
|
||||
# PyTorch setup
|
||||
self.use_torch = TORCH_AVAILABLE
|
||||
if self.use_torch:
|
||||
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
|
||||
hidden_dim = config.get('hidden_dim', 128)
|
||||
self.policy_net = TradingNetwork(state_dim, action_dim, hidden_dim).to(self.device)
|
||||
self.target_net = TradingNetwork(state_dim, action_dim, hidden_dim).to(self.device)
|
||||
self.target_net.load_state_dict(self.policy_net.state_dict())
|
||||
self.target_net.eval()
|
||||
self.optimizer = optim.Adam(self.policy_net.parameters(), lr=self.learning_rate)
|
||||
logger.info(f"RL Agent initialized (PyTorch, device={self.device}, "
|
||||
f"state_dim={state_dim}, action_dim={action_dim})")
|
||||
else:
|
||||
self.device = None
|
||||
self.policy_net = None
|
||||
self.target_net = None
|
||||
self.optimizer = None
|
||||
logger.info("RL Agent initialized (random mode - no PyTorch)")
|
||||
|
||||
def select_action(self, state: np.ndarray, live_mode: bool = False) -> int:
|
||||
"""
|
||||
Epsilon-greedy action selection.
|
||||
In live mode, uses live_epsilon instead of training epsilon.
|
||||
"""
|
||||
eps = self.live_epsilon if live_mode else self.epsilon
|
||||
|
||||
if random.random() < eps:
|
||||
return random.randint(0, self.action_dim - 1)
|
||||
|
||||
if not self.use_torch:
|
||||
return random.randint(0, self.action_dim - 1)
|
||||
|
||||
with torch.no_grad():
|
||||
state_tensor = torch.FloatTensor(state).unsqueeze(0).to(self.device)
|
||||
q_values = self.policy_net(state_tensor)
|
||||
return int(q_values.argmax(dim=1).item())
|
||||
|
||||
def store_experience(self, state, action, reward, next_state, done):
|
||||
"""Store transition in replay buffer"""
|
||||
self.memory.append((state, action, reward, next_state, done))
|
||||
|
||||
def train_step(self) -> Optional[float]:
|
||||
"""
|
||||
Sample mini-batch from replay buffer, compute DQN loss, update.
|
||||
Returns loss value or None if not enough samples.
|
||||
"""
|
||||
if not self.use_torch:
|
||||
return None
|
||||
|
||||
if len(self.memory) < self.batch_size:
|
||||
return None
|
||||
|
||||
batch = random.sample(self.memory, self.batch_size)
|
||||
states, actions, rewards, next_states, dones = zip(*batch)
|
||||
|
||||
states = torch.FloatTensor(np.array(states)).to(self.device)
|
||||
actions = torch.LongTensor(actions).to(self.device)
|
||||
rewards = torch.FloatTensor(rewards).to(self.device)
|
||||
next_states = torch.FloatTensor(np.array(next_states)).to(self.device)
|
||||
dones = torch.BoolTensor(dones).to(self.device)
|
||||
|
||||
# Current Q values
|
||||
current_q = self.policy_net(states).gather(1, actions.unsqueeze(1)).squeeze(1)
|
||||
|
||||
# Target Q values
|
||||
with torch.no_grad():
|
||||
next_q = self.target_net(next_states).max(1)[0]
|
||||
next_q[dones] = 0.0
|
||||
target_q = rewards + self.gamma * next_q
|
||||
|
||||
# Loss and backprop
|
||||
loss = nn.functional.smooth_l1_loss(current_q, target_q)
|
||||
self.optimizer.zero_grad()
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(self.policy_net.parameters(), 1.0)
|
||||
self.optimizer.step()
|
||||
|
||||
self.steps += 1
|
||||
|
||||
# Update target network
|
||||
if self.steps % self.target_update_freq == 0:
|
||||
self.target_net.load_state_dict(self.policy_net.state_dict())
|
||||
|
||||
# Decay epsilon
|
||||
self.epsilon = max(self.epsilon_min, self.epsilon * self.epsilon_decay)
|
||||
|
||||
loss_val = loss.item()
|
||||
self.training_losses.append(loss_val)
|
||||
|
||||
return loss_val
|
||||
|
||||
def train_on_episode(self, env, candles_df, ga_signal_fn=None) -> Dict:
|
||||
"""
|
||||
Train on a full episode (backtest run through candles).
|
||||
Returns training metrics.
|
||||
"""
|
||||
state = env.reset(candles_df)
|
||||
if state is None:
|
||||
return {'avg_loss': 0, 'total_reward': 0, 'steps': 0}
|
||||
|
||||
total_reward = 0
|
||||
total_loss = 0
|
||||
loss_count = 0
|
||||
step = 0
|
||||
done = False
|
||||
|
||||
while not done:
|
||||
# Get GA signal if available
|
||||
ga_signal = None
|
||||
if ga_signal_fn and step < len(candles_df):
|
||||
ga_signal = ga_signal_fn(step)
|
||||
|
||||
action = self.select_action(state)
|
||||
next_state, reward, done, info = env.step(action, ga_signal=ga_signal)
|
||||
|
||||
self.store_experience(state, action, reward, next_state, done)
|
||||
|
||||
loss = self.train_step()
|
||||
if loss is not None:
|
||||
total_loss += loss
|
||||
loss_count += 1
|
||||
|
||||
total_reward += reward
|
||||
state = next_state
|
||||
step += 1
|
||||
|
||||
avg_loss = total_loss / max(loss_count, 1)
|
||||
|
||||
return {
|
||||
'avg_loss': round(avg_loss, 6),
|
||||
'total_reward': round(total_reward, 4),
|
||||
'steps': step,
|
||||
'epsilon': round(self.epsilon, 4),
|
||||
'total_trades': env.total_trades,
|
||||
'total_pnl': round(env.total_pnl, 4),
|
||||
'final_equity': round(env.equity_history[-1] if env.equity_history else 0, 2),
|
||||
'memory_size': len(self.memory),
|
||||
}
|
||||
|
||||
def update_from_live_trade(self, state, action, reward, next_state):
|
||||
"""
|
||||
Online learning: called after each live trade result.
|
||||
Stores experience and does one training step.
|
||||
"""
|
||||
self.store_experience(state, action, reward, next_state, False)
|
||||
self.train_step()
|
||||
|
||||
def save(self, store, epoch: int = None):
|
||||
"""Save model checkpoint to DataStore"""
|
||||
if not self.use_torch:
|
||||
return
|
||||
|
||||
if epoch is None:
|
||||
epoch = self.steps
|
||||
|
||||
state_bytes = self._get_state_dict_bytes()
|
||||
metrics = {
|
||||
'epsilon': self.epsilon,
|
||||
'steps': self.steps,
|
||||
'memory_size': len(self.memory),
|
||||
'avg_loss': round(np.mean(self.training_losses[-100:]), 6)
|
||||
if self.training_losses else 0,
|
||||
}
|
||||
store.save_model_checkpoint('rl_agent', epoch, state_bytes, metrics)
|
||||
|
||||
# Also save experience replay memory
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
db_path_str = str(store.db_path)
|
||||
memory_path = db_path_str.replace('.db', '_rl_memory.pkl')
|
||||
try:
|
||||
with open(memory_path, 'wb') as f:
|
||||
# Save memory as list to avoid deque pickle issues
|
||||
pickle.dump(list(self.memory), f)
|
||||
logger.debug(f"RL memory saved ({len(self.memory)} experiences)")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to save RL memory: {e}")
|
||||
|
||||
logger.info(f"RL model saved (epoch {epoch}, epsilon={self.epsilon:.4f})")
|
||||
|
||||
def load(self, store) -> bool:
|
||||
"""Load latest checkpoint from DataStore"""
|
||||
if not self.use_torch:
|
||||
return False
|
||||
|
||||
checkpoint = store.load_latest_checkpoint('rl_agent')
|
||||
if checkpoint is None:
|
||||
logger.info("No RL checkpoint found, starting fresh")
|
||||
return False
|
||||
|
||||
try:
|
||||
buffer = io.BytesIO(checkpoint['state_dict'])
|
||||
state_dict = torch.load(buffer, map_location=self.device, weights_only=True)
|
||||
|
||||
# Check dimension compatibility before loading
|
||||
first_layer_key = 'net.0.weight'
|
||||
if first_layer_key in state_dict:
|
||||
saved_input_dim = state_dict[first_layer_key].shape[1]
|
||||
if saved_input_dim != self.state_dim:
|
||||
logger.warning(f"RL checkpoint dimension mismatch (saved={saved_input_dim}, "
|
||||
f"current={self.state_dim}). Starting fresh with new architecture.")
|
||||
return False
|
||||
|
||||
self.policy_net.load_state_dict(state_dict)
|
||||
self.target_net.load_state_dict(state_dict)
|
||||
|
||||
import json
|
||||
metrics = json.loads(checkpoint.get('metrics', '{}'))
|
||||
self.epsilon = metrics.get('epsilon', self.epsilon)
|
||||
self.steps = metrics.get('steps', self.steps)
|
||||
|
||||
# Load experience replay memory
|
||||
import pickle
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
db_path_str = str(store.db_path)
|
||||
memory_path = db_path_str.replace('.db', '_rl_memory.pkl')
|
||||
try:
|
||||
with open(memory_path, 'rb') as f:
|
||||
saved_memory = pickle.load(f)
|
||||
self.memory = deque(saved_memory, maxlen=self.config['memory_size'])
|
||||
logger.info(f"RL memory loaded ({len(self.memory)} experiences)")
|
||||
except FileNotFoundError:
|
||||
logger.debug("No saved RL memory found, starting with empty buffer")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load RL memory: {e}")
|
||||
|
||||
logger.info(f"RL model loaded (epoch {checkpoint['epoch']}, "
|
||||
f"epsilon={self.epsilon:.4f})")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not load RL checkpoint (likely dimension change): {e}. Starting fresh.")
|
||||
return False
|
||||
|
||||
def _get_state_dict_bytes(self) -> bytes:
|
||||
"""Serialize model state dict to bytes"""
|
||||
buffer = io.BytesIO()
|
||||
torch.save(self.policy_net.state_dict(), buffer)
|
||||
return buffer.getvalue()
|
||||
|
||||
def get_stats(self) -> Dict:
|
||||
"""Get current agent statistics"""
|
||||
return {
|
||||
'epsilon': round(self.epsilon, 4),
|
||||
'steps': self.steps,
|
||||
'memory_size': len(self.memory),
|
||||
'avg_loss': round(np.mean(self.training_losses[-100:]), 6)
|
||||
if self.training_losses else 0,
|
||||
'device': str(self.device) if self.device else 'random',
|
||||
'use_torch': self.use_torch,
|
||||
}
|
||||
+74
-14
@@ -44,8 +44,13 @@ class TradingEnvironment:
|
||||
self.portfolio_features = 3
|
||||
# GA signal features: [signal_direction, confidence]
|
||||
self.ga_features = 2
|
||||
# Market regime features: [regime_code, trend_strength_normalized]
|
||||
self.regime_features = 2
|
||||
|
||||
self.state_dim = FeatureEngine.NUM_FEATURES + self.portfolio_features + self.ga_features
|
||||
self.state_dim = (FeatureEngine.NUM_FEATURES
|
||||
+ self.portfolio_features
|
||||
+ self.ga_features
|
||||
+ self.regime_features) # = 36
|
||||
|
||||
# State tracking
|
||||
self.reset()
|
||||
@@ -65,6 +70,7 @@ class TradingEnvironment:
|
||||
self.total_pnl = 0.0
|
||||
self.wins = 0
|
||||
self.losses = 0
|
||||
self._last_side = None # Track last position side for flip bonus
|
||||
|
||||
self.candles_df = candles_df
|
||||
self.features_df = None
|
||||
@@ -185,6 +191,7 @@ class TradingEnvironment:
|
||||
# CLOSE POSITION (long or short)
|
||||
elif action == self.CLOSE_HALF and self.position_shares != 0:
|
||||
close_shares = abs(self.position_shares) * 0.5
|
||||
closed_side = 'long' if self.position_shares > 0 else 'short'
|
||||
if self.position_shares > 0:
|
||||
# Close half of long
|
||||
proceeds = close_shares * current_price
|
||||
@@ -210,9 +217,13 @@ class TradingEnvironment:
|
||||
self.last_action_step = self.step_count
|
||||
info['trade'] = 'close_50%'
|
||||
info['pnl'] = pnl
|
||||
# Track for flip bonus
|
||||
if self.position_shares == 0:
|
||||
self._last_side = closed_side
|
||||
|
||||
elif action == self.CLOSE_ALL and self.position_shares != 0:
|
||||
abs_shares = abs(self.position_shares)
|
||||
closed_side = 'long' if self.position_shares > 0 else 'short'
|
||||
if self.position_shares > 0:
|
||||
# Close all long
|
||||
proceeds = abs_shares * current_price
|
||||
@@ -226,6 +237,7 @@ class TradingEnvironment:
|
||||
pnl = (self.position_price - current_price) * abs_shares - fees
|
||||
self.capital += (self.position_price * abs_shares) - cost - fees
|
||||
|
||||
self._last_side = closed_side
|
||||
self.position_shares = 0
|
||||
self.position_price = 0
|
||||
self.total_trades += 1
|
||||
@@ -272,26 +284,57 @@ class TradingEnvironment:
|
||||
def _compute_reward(self, action: int, prev_equity: float,
|
||||
curr_equity: float, current_price: float) -> float:
|
||||
"""
|
||||
Reward function for scalping:
|
||||
Enhanced reward function for scalping with drawdown/directional protection:
|
||||
- Base: portfolio return (amplified for scalping sensitivity)
|
||||
- Penalty: drawdown, overtrading
|
||||
- Penalty: extended drawdowns (>3% from peak)
|
||||
- Penalty: over-levered positions (position > 40% of equity)
|
||||
- Penalty: extended one-directional bets (same direction > 20 steps)
|
||||
- Bonus: profitable close (long or short)
|
||||
- Scalp bonus: quick profitable round trips
|
||||
"""
|
||||
if prev_equity <= 0:
|
||||
return 0.0
|
||||
|
||||
# PROFIT IS KING. Everything else is noise.
|
||||
|
||||
# Base reward: actual P&L change (amplified for sensitivity)
|
||||
# ─── 1. Base P&L reward ───────────────────────────────────────────────
|
||||
pnl_change = curr_equity - prev_equity
|
||||
base_reward = (pnl_change / prev_equity) * 5.0
|
||||
|
||||
# Drawdown penalty (only kick in at 5%+, don't punish normal swings)
|
||||
# ─── 2. Extended drawdown penalty ─────────────────────────────────────
|
||||
# Only penalize after 3% drawdown (allow normal volatility)
|
||||
drawdown = (self.peak_equity - curr_equity) / self.peak_equity if self.peak_equity > 0 else 0
|
||||
dd_penalty = -1.0 * max(0, drawdown - 0.05)
|
||||
dd_penalty = 0.0
|
||||
if drawdown > 0.03:
|
||||
# Escalating penalty: -0.02 per 1% beyond 3%
|
||||
dd_penalty = -0.02 * ((drawdown - 0.03) * 100)
|
||||
elif drawdown > 0.10:
|
||||
# Severe penalty for deep drawdowns (>10%)
|
||||
dd_penalty = -0.05 * ((drawdown - 0.10) * 100)
|
||||
|
||||
# Profitable close bonus — the BIGGEST reward signal
|
||||
# ─── 3. Over-leverage penalty ─────────────────────────────────────────
|
||||
leverage_penalty = 0.0
|
||||
if self.position_shares != 0 and self.position_price > 0:
|
||||
position_value = abs(self.position_shares * self.position_price)
|
||||
leverage_ratio = position_value / prev_equity if prev_equity > 0 else 0
|
||||
if leverage_ratio > 0.40:
|
||||
# Penalty for >40% position size
|
||||
leverage_penalty = -0.02 * ((leverage_ratio - 0.40) * 100)
|
||||
elif leverage_ratio > 0.60:
|
||||
# Severe penalty for >60% (dangerous over-leverage)
|
||||
leverage_penalty = -0.05 * ((leverage_ratio - 0.60) * 100)
|
||||
|
||||
# ─── 4. Extended one-directional bet penalty ─────────────────────────
|
||||
# Track how long we've been holding the SAME direction without closing
|
||||
directional_penalty = 0.0
|
||||
if self.position_shares != 0:
|
||||
hold_time = self.step_count - self.entry_step
|
||||
if hold_time > 20:
|
||||
# After 20 candles (≈100 min on 5m), start penalizing prolonged holds
|
||||
directional_penalty = -0.005 * (hold_time - 20)
|
||||
if hold_time > 40:
|
||||
# After 40 candles, severe penalty for stubborn directional bets
|
||||
directional_penalty = -0.015 * (hold_time - 40)
|
||||
|
||||
# ─── 5. Profitable close bonus ───────────────────────────────────────
|
||||
close_bonus = 0.0
|
||||
if action in (self.CLOSE_HALF, self.CLOSE_ALL) and self.position_price > 0:
|
||||
if self.position_shares > 0 and current_price > self.position_price:
|
||||
@@ -307,17 +350,31 @@ class TradingEnvironment:
|
||||
pnl_pct = (current_price - self.position_price) / self.position_price
|
||||
close_bonus = -0.05 - pnl_pct * 3.0
|
||||
|
||||
# Quick scalp bonus: fast profitable round trips get extra reward
|
||||
# ─── 6. Quick scalp bonus ─────────────────────────────────────────────
|
||||
scalp_bonus = 0.0
|
||||
if action in (self.CLOSE_HALF, self.CLOSE_ALL) and close_bonus > 0:
|
||||
hold_time = self.step_count - self.entry_step
|
||||
if hold_time < 12:
|
||||
scalp_bonus = 0.02
|
||||
|
||||
# NO hold penalty — sitting in cash when uncertain is SMART
|
||||
# NO overtrading penalty — let the agent trade freely
|
||||
# ─── 7. Regime-aware directional flip bonus ───────────────────────────
|
||||
# If we've been short and now buy (or vice versa), small bonus for adapting
|
||||
flip_bonus = 0.0
|
||||
if action in (self.BUY_SMALL, self.BUY_LARGE) and self.position_shares == 0:
|
||||
# Was flat — check if we just closed a short profitably (market reversal)
|
||||
if hasattr(self, '_last_side') and self._last_side == 'short':
|
||||
flip_bonus = 0.03 # Bonus for correctly reversing from short to long
|
||||
elif action in (self.SHORT_SMALL, self.SHORT_LARGE) and self.position_shares == 0:
|
||||
if hasattr(self, '_last_side') and self._last_side == 'long':
|
||||
flip_bonus = 0.03 # Bonus for correctly reversing from long to short
|
||||
|
||||
reward = base_reward + dd_penalty + close_bonus + scalp_bonus
|
||||
reward = (base_reward
|
||||
+ dd_penalty
|
||||
+ leverage_penalty
|
||||
+ directional_penalty
|
||||
+ close_bonus
|
||||
+ scalp_bonus
|
||||
+ flip_bonus)
|
||||
|
||||
# Clip to [-1, 1]
|
||||
return max(-1.0, min(1.0, reward))
|
||||
@@ -382,7 +439,10 @@ class TradingEnvironment:
|
||||
|
||||
ga_features = np.array([signal_dir, confidence], dtype=np.float32)
|
||||
|
||||
return np.concatenate([market_features, portfolio_features, ga_features])
|
||||
# Regime features (placeholders for backward compatibility; real values set by main_auto)
|
||||
regime_features = np.array([0.0, 0.0], dtype=np.float32)
|
||||
|
||||
return np.concatenate([market_features, portfolio_features, ga_features, regime_features])
|
||||
|
||||
def get_portfolio_state(self, current_price: float = 0) -> Dict:
|
||||
"""Get portfolio state dict for external use (handles long and short)"""
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
"""
|
||||
Trading Environment for Reinforcement Learning
|
||||
Defines state space, action space, and reward function.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from typing import Dict, Tuple, Optional
|
||||
from loguru import logger
|
||||
|
||||
from data.features import FeatureEngine
|
||||
|
||||
|
||||
class TradingEnvironment:
|
||||
"""
|
||||
Trading environment for RL agent.
|
||||
Modes:
|
||||
- 'backtest': steps through historical candles
|
||||
- 'live': receives state updates from the trading loop
|
||||
"""
|
||||
|
||||
# Actions
|
||||
HOLD = 0
|
||||
BUY_SMALL = 1 # Buy with 25% of available capital (go long)
|
||||
BUY_LARGE = 2 # Buy with 50% of available capital (go long)
|
||||
CLOSE_HALF = 3 # Close 50% of position (long or short)
|
||||
CLOSE_ALL = 4 # Close 100% of position (long or short)
|
||||
SHORT_SMALL = 5 # Short with 25% of available capital
|
||||
SHORT_LARGE = 6 # Short with 50% of available capital
|
||||
|
||||
ACTION_NAMES = ['hold', 'buy_25%', 'buy_50%', 'close_50%', 'close_all',
|
||||
'short_25%', 'short_50%']
|
||||
NUM_ACTIONS = 7
|
||||
|
||||
def __init__(self, feature_engine: FeatureEngine,
|
||||
initial_capital: float = 100.0,
|
||||
commission_rate: float = 0.001):
|
||||
self.feature_engine = feature_engine
|
||||
self.initial_capital = initial_capital
|
||||
self.commission_rate = commission_rate
|
||||
|
||||
# Portfolio features appended to market features
|
||||
# [position_ratio, unrealized_pnl_pct, time_in_position_normalized]
|
||||
self.portfolio_features = 3
|
||||
# GA signal features: [signal_direction, confidence]
|
||||
self.ga_features = 2
|
||||
|
||||
self.state_dim = FeatureEngine.NUM_FEATURES + self.portfolio_features + self.ga_features
|
||||
|
||||
# State tracking
|
||||
self.reset()
|
||||
|
||||
def reset(self, candles_df: pd.DataFrame = None) -> Optional[np.ndarray]:
|
||||
"""Reset environment for a new episode"""
|
||||
self.capital = self.initial_capital
|
||||
self.position_shares = 0.0
|
||||
self.position_price = 0.0
|
||||
self.step_count = 0
|
||||
self.entry_step = 0
|
||||
self.equity_history = [self.initial_capital]
|
||||
self.peak_equity = self.initial_capital
|
||||
self.last_action = self.HOLD
|
||||
self.last_action_step = -10
|
||||
self.total_trades = 0
|
||||
self.total_pnl = 0.0
|
||||
self.wins = 0
|
||||
self.losses = 0
|
||||
|
||||
self.candles_df = candles_df
|
||||
self.features_df = None
|
||||
|
||||
if candles_df is not None and len(candles_df) > 50:
|
||||
self.features_df = self.feature_engine.compute_and_normalize(candles_df)
|
||||
if len(self.features_df) > 0:
|
||||
return self._get_state(0)
|
||||
|
||||
return np.zeros(self.state_dim, dtype=np.float32)
|
||||
|
||||
def step(self, action: int, ga_signal: Dict = None,
|
||||
current_candle: pd.Series = None) -> Tuple[np.ndarray, float, bool, Dict]:
|
||||
"""
|
||||
Execute action, advance one timestep.
|
||||
|
||||
Args:
|
||||
action: integer action (0-4)
|
||||
ga_signal: optional GA strategy signal {signal, confidence}
|
||||
current_candle: optional candle for live mode
|
||||
|
||||
Returns: (next_state, reward, done, info)
|
||||
"""
|
||||
prev_equity = self._get_equity()
|
||||
|
||||
# Get current price
|
||||
if self.features_df is not None and self.step_count < len(self.features_df):
|
||||
idx = self.step_count
|
||||
current_price = float(self.candles_df['close'].iloc[
|
||||
self.candles_df.index.get_indexer(
|
||||
[self.features_df.index[idx]], method='nearest'
|
||||
)[0]
|
||||
]) if self.candles_df is not None else 0
|
||||
# Simpler: just use the close from the original df aligned by position
|
||||
try:
|
||||
orig_idx = self.features_df.index[idx]
|
||||
if orig_idx in self.candles_df.index:
|
||||
current_price = float(self.candles_df.loc[orig_idx, 'close'])
|
||||
else:
|
||||
current_price = float(self.candles_df['close'].iloc[-1])
|
||||
except (IndexError, KeyError):
|
||||
current_price = float(self.candles_df['close'].iloc[-1])
|
||||
elif current_candle is not None:
|
||||
current_price = float(current_candle.get('close', current_candle.get('Close', 0)))
|
||||
else:
|
||||
return np.zeros(self.state_dim, dtype=np.float32), 0.0, True, {}
|
||||
|
||||
# Execute action
|
||||
trade_info = self._execute_action(action, current_price)
|
||||
|
||||
self.step_count += 1
|
||||
curr_equity = self._get_equity(current_price)
|
||||
self.equity_history.append(curr_equity)
|
||||
|
||||
if curr_equity > self.peak_equity:
|
||||
self.peak_equity = curr_equity
|
||||
|
||||
# Compute reward
|
||||
reward = self._compute_reward(action, prev_equity, curr_equity, current_price)
|
||||
|
||||
# Check if done
|
||||
done = False
|
||||
if self.features_df is not None:
|
||||
done = self.step_count >= len(self.features_df) - 1
|
||||
if curr_equity < self.initial_capital * 0.5: # 50% loss = episode over
|
||||
done = True
|
||||
|
||||
# Get next state
|
||||
next_state = self._get_state(self.step_count, ga_signal)
|
||||
|
||||
info = {
|
||||
'equity': curr_equity,
|
||||
'position_value': abs(self.position_shares) * current_price if self.position_shares != 0 else 0,
|
||||
'position_side': 'long' if self.position_shares > 0 else ('short' if self.position_shares < 0 else 'flat'),
|
||||
'capital': self.capital,
|
||||
'total_trades': self.total_trades,
|
||||
'total_pnl': self.total_pnl,
|
||||
**trade_info,
|
||||
}
|
||||
|
||||
return next_state, reward, done, info
|
||||
|
||||
def _execute_action(self, action: int, current_price: float) -> Dict:
|
||||
"""Execute a trading action, return trade info.
|
||||
position_shares > 0 means long, < 0 means short."""
|
||||
info = {'trade': None}
|
||||
|
||||
if current_price <= 0:
|
||||
return info
|
||||
|
||||
# GO LONG (only if flat)
|
||||
if action == self.BUY_SMALL and self.position_shares == 0:
|
||||
invest = self.capital * 0.25
|
||||
if invest > 1:
|
||||
fees = invest * self.commission_rate
|
||||
shares = (invest - fees) / current_price
|
||||
self.capital -= invest
|
||||
self.position_shares = shares
|
||||
self.position_price = current_price
|
||||
self.entry_step = self.step_count
|
||||
self.last_action = action
|
||||
self.last_action_step = self.step_count
|
||||
info['trade'] = 'buy_25%'
|
||||
|
||||
elif action == self.BUY_LARGE and self.position_shares == 0:
|
||||
invest = self.capital * 0.50
|
||||
if invest > 1:
|
||||
fees = invest * self.commission_rate
|
||||
shares = (invest - fees) / current_price
|
||||
self.capital -= invest
|
||||
self.position_shares = shares
|
||||
self.position_price = current_price
|
||||
self.entry_step = self.step_count
|
||||
self.last_action = action
|
||||
self.last_action_step = self.step_count
|
||||
info['trade'] = 'buy_50%'
|
||||
|
||||
# CLOSE POSITION (long or short)
|
||||
elif action == self.CLOSE_HALF and self.position_shares != 0:
|
||||
close_shares = abs(self.position_shares) * 0.5
|
||||
if self.position_shares > 0:
|
||||
# Close half of long
|
||||
proceeds = close_shares * current_price
|
||||
fees = proceeds * self.commission_rate
|
||||
pnl = (current_price - self.position_price) * close_shares - fees
|
||||
self.capital += proceeds - fees
|
||||
self.position_shares -= close_shares
|
||||
else:
|
||||
# Cover half of short
|
||||
cost = close_shares * current_price
|
||||
fees = cost * self.commission_rate
|
||||
pnl = (self.position_price - current_price) * close_shares - fees
|
||||
self.capital += (self.position_price * close_shares) - cost - fees
|
||||
self.position_shares += close_shares
|
||||
|
||||
self.total_trades += 1
|
||||
self.total_pnl += pnl
|
||||
if pnl > 0:
|
||||
self.wins += 1
|
||||
else:
|
||||
self.losses += 1
|
||||
self.last_action = action
|
||||
self.last_action_step = self.step_count
|
||||
info['trade'] = 'close_50%'
|
||||
info['pnl'] = pnl
|
||||
|
||||
elif action == self.CLOSE_ALL and self.position_shares != 0:
|
||||
abs_shares = abs(self.position_shares)
|
||||
if self.position_shares > 0:
|
||||
# Close all long
|
||||
proceeds = abs_shares * current_price
|
||||
fees = proceeds * self.commission_rate
|
||||
pnl = (current_price - self.position_price) * abs_shares - fees
|
||||
self.capital += proceeds - fees
|
||||
else:
|
||||
# Cover all short
|
||||
cost = abs_shares * current_price
|
||||
fees = cost * self.commission_rate
|
||||
pnl = (self.position_price - current_price) * abs_shares - fees
|
||||
self.capital += (self.position_price * abs_shares) - cost - fees
|
||||
|
||||
self.position_shares = 0
|
||||
self.position_price = 0
|
||||
self.total_trades += 1
|
||||
self.total_pnl += pnl
|
||||
if pnl > 0:
|
||||
self.wins += 1
|
||||
else:
|
||||
self.losses += 1
|
||||
self.last_action = action
|
||||
self.last_action_step = self.step_count
|
||||
info['trade'] = 'close_all'
|
||||
info['pnl'] = pnl
|
||||
|
||||
# GO SHORT (only if flat)
|
||||
elif action == self.SHORT_SMALL and self.position_shares == 0:
|
||||
invest = self.capital * 0.25
|
||||
if invest > 1:
|
||||
fees = invest * self.commission_rate
|
||||
shares = (invest - fees) / current_price
|
||||
# Short: we receive proceeds upfront, owe shares later
|
||||
self.capital += invest - fees # margin collateral stays, proceeds added
|
||||
self.position_shares = -shares
|
||||
self.position_price = current_price
|
||||
self.entry_step = self.step_count
|
||||
self.last_action = action
|
||||
self.last_action_step = self.step_count
|
||||
info['trade'] = 'short_25%'
|
||||
|
||||
elif action == self.SHORT_LARGE and self.position_shares == 0:
|
||||
invest = self.capital * 0.50
|
||||
if invest > 1:
|
||||
fees = invest * self.commission_rate
|
||||
shares = (invest - fees) / current_price
|
||||
self.capital += invest - fees
|
||||
self.position_shares = -shares
|
||||
self.position_price = current_price
|
||||
self.entry_step = self.step_count
|
||||
self.last_action = action
|
||||
self.last_action_step = self.step_count
|
||||
info['trade'] = 'short_50%'
|
||||
|
||||
return info
|
||||
|
||||
def _compute_reward(self, action: int, prev_equity: float,
|
||||
curr_equity: float, current_price: float) -> float:
|
||||
"""
|
||||
Reward function for scalping:
|
||||
- Base: portfolio return (amplified for scalping sensitivity)
|
||||
- Penalty: drawdown, overtrading
|
||||
- Bonus: profitable close (long or short)
|
||||
- Scalp bonus: quick profitable round trips
|
||||
"""
|
||||
if prev_equity <= 0:
|
||||
return 0.0
|
||||
|
||||
# Base reward: portfolio return (amplified 2x for scalping sensitivity)
|
||||
base_reward = (curr_equity - prev_equity) / prev_equity * 2.0
|
||||
|
||||
# Drawdown penalty
|
||||
drawdown = (self.peak_equity - curr_equity) / self.peak_equity if self.peak_equity > 0 else 0
|
||||
dd_penalty = -0.5 * max(0, drawdown - 0.03)
|
||||
|
||||
# Overtrading penalty (reduced for scalping - allow faster re-entry)
|
||||
overtrade_penalty = 0.0
|
||||
if action != self.HOLD and (self.step_count - self.last_action_step) < 2:
|
||||
overtrade_penalty = -0.0005
|
||||
|
||||
# Holding penalty (stronger for scalping - don't sit idle)
|
||||
hold_penalty = 0.0
|
||||
if action == self.HOLD and self.position_shares == 0:
|
||||
hold_penalty = -0.0002
|
||||
|
||||
# Profitable close bonus (works for both long and short)
|
||||
close_bonus = 0.0
|
||||
if action in (self.CLOSE_HALF, self.CLOSE_ALL) and self.position_price > 0:
|
||||
if self.position_shares > 0 and current_price > self.position_price:
|
||||
# Profitable long close
|
||||
pnl_pct = (current_price - self.position_price) / self.position_price
|
||||
close_bonus = 0.02 * pnl_pct
|
||||
elif self.position_shares < 0 and current_price < self.position_price:
|
||||
# Profitable short close
|
||||
pnl_pct = (self.position_price - current_price) / self.position_price
|
||||
close_bonus = 0.02 * pnl_pct
|
||||
|
||||
# Quick scalp bonus: reward fast profitable round trips
|
||||
scalp_bonus = 0.0
|
||||
if action in (self.CLOSE_HALF, self.CLOSE_ALL) and self.position_shares != 0:
|
||||
hold_time = self.step_count - self.entry_step
|
||||
if hold_time < 12 and close_bonus > 0: # Quick + profitable
|
||||
scalp_bonus = 0.005
|
||||
|
||||
reward = base_reward + dd_penalty + overtrade_penalty + hold_penalty + close_bonus + scalp_bonus
|
||||
|
||||
# Clip to [-1, 1]
|
||||
return max(-1.0, min(1.0, reward))
|
||||
|
||||
def _get_equity(self, current_price: float = None) -> float:
|
||||
"""Calculate current total equity (handles long and short positions)"""
|
||||
if current_price is None:
|
||||
if self.candles_df is not None and self.step_count < len(self.candles_df):
|
||||
current_price = float(self.candles_df['close'].iloc[self.step_count])
|
||||
else:
|
||||
current_price = self.position_price if self.position_price > 0 else 0
|
||||
|
||||
if self.position_shares > 0:
|
||||
# Long: capital + shares * price
|
||||
return self.capital + self.position_shares * current_price
|
||||
elif self.position_shares < 0:
|
||||
# Short: capital + unrealized P&L from short
|
||||
abs_shares = abs(self.position_shares)
|
||||
short_pnl = (self.position_price - current_price) * abs_shares
|
||||
return self.capital + short_pnl
|
||||
return self.capital
|
||||
|
||||
def _get_state(self, step: int, ga_signal: Dict = None) -> np.ndarray:
|
||||
"""Build full state vector"""
|
||||
# Market features
|
||||
if self.features_df is not None and step < len(self.features_df):
|
||||
market_features = self.feature_engine.get_state_vector(self.features_df, step)
|
||||
else:
|
||||
market_features = np.zeros(FeatureEngine.NUM_FEATURES, dtype=np.float32)
|
||||
|
||||
# Portfolio features (handles long and short)
|
||||
equity = self._get_equity()
|
||||
position_value = abs(self.position_shares) * self.position_price
|
||||
# Positive ratio = long, negative ratio = short
|
||||
position_ratio = (self.position_shares * self.position_price) / equity if equity > 0 else 0.0
|
||||
|
||||
unrealized_pnl = 0.0
|
||||
if self.position_shares != 0 and self.position_price > 0:
|
||||
if self.candles_df is not None and step < len(self.candles_df):
|
||||
current = float(self.candles_df['close'].iloc[min(step, len(self.candles_df) - 1)])
|
||||
if self.position_shares > 0:
|
||||
unrealized_pnl = (current - self.position_price) / self.position_price
|
||||
else:
|
||||
unrealized_pnl = (self.position_price - current) / self.position_price
|
||||
|
||||
time_in_position = 0.0
|
||||
if self.position_shares != 0:
|
||||
time_in_position = min((self.step_count - self.entry_step) / 48.0, 1.0)
|
||||
|
||||
portfolio_features = np.array([
|
||||
position_ratio, unrealized_pnl, time_in_position
|
||||
], dtype=np.float32)
|
||||
|
||||
# GA signal features
|
||||
if ga_signal:
|
||||
signal_dir = 1.0 if ga_signal.get('signal') == 'buy' else (
|
||||
-1.0 if ga_signal.get('signal') == 'sell' else 0.0)
|
||||
confidence = float(ga_signal.get('confidence', 0.0))
|
||||
else:
|
||||
signal_dir = 0.0
|
||||
confidence = 0.0
|
||||
|
||||
ga_features = np.array([signal_dir, confidence], dtype=np.float32)
|
||||
|
||||
return np.concatenate([market_features, portfolio_features, ga_features])
|
||||
|
||||
def get_portfolio_state(self, current_price: float = 0) -> Dict:
|
||||
"""Get portfolio state dict for external use (handles long and short)"""
|
||||
equity = self._get_equity(current_price)
|
||||
position_value = self.position_shares * current_price if current_price > 0 else 0
|
||||
unrealized = 0.0
|
||||
if self.position_price > 0 and current_price > 0 and self.position_shares != 0:
|
||||
if self.position_shares > 0:
|
||||
unrealized = (current_price - self.position_price) / self.position_price
|
||||
else:
|
||||
unrealized = (self.position_price - current_price) / self.position_price
|
||||
return {
|
||||
'position_ratio': position_value / equity if equity > 0 else 0,
|
||||
'unrealized_pnl': unrealized,
|
||||
'time_in_position': min((self.step_count - self.entry_step) / 48.0, 1.0)
|
||||
if self.position_shares != 0 else 0,
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
"""
|
||||
Auto Strategy
|
||||
Converts a StrategyGenome into a callable trading strategy for backtesting and live trading.
|
||||
|
||||
Optimized: indicators are pre-computed once as arrays, not per-candle.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from typing import Dict, Callable
|
||||
from loguru import logger
|
||||
|
||||
|
||||
def genome_to_strategy(genome) -> Callable:
|
||||
"""
|
||||
Convert a StrategyGenome into a scalping strategy with long AND short signals.
|
||||
|
||||
Returns a callable with signature:
|
||||
(state: Dict, candle_idx: int, df: pd.DataFrame, params: Dict) -> Dict
|
||||
|
||||
Indicators are pre-computed on first call and cached via closure.
|
||||
"""
|
||||
g = genome
|
||||
_cache = {}
|
||||
|
||||
def strategy_fn(state: Dict, idx: int, df: pd.DataFrame, params: Dict) -> Dict:
|
||||
"""Evaluate scalping strategy at candle index"""
|
||||
if idx < max(g.slow_ma_period, 30):
|
||||
return {'action': 'hold'}
|
||||
|
||||
# Pre-compute all indicators once, cache by DataFrame id
|
||||
df_id = id(df)
|
||||
if df_id not in _cache:
|
||||
_cache.clear()
|
||||
_cache[df_id] = _precompute_indicators(df, g)
|
||||
|
||||
ind = _cache[df_id]
|
||||
|
||||
if idx >= len(ind['fast_ma']):
|
||||
return {'action': 'hold'}
|
||||
|
||||
current_price = ind['close'][idx]
|
||||
fast_ma = ind['fast_ma'][idx]
|
||||
slow_ma = ind['slow_ma'][idx]
|
||||
rsi = ind['rsi'][idx]
|
||||
macd_hist = ind['macd_hist'][idx]
|
||||
atr = ind['atr'][idx]
|
||||
vol_ratio = ind['vol_ratio'][idx]
|
||||
|
||||
# BB for mean-reversion scalps
|
||||
bb_upper = ind.get('bb_upper')
|
||||
bb_lower = ind.get('bb_lower')
|
||||
|
||||
position = state.get('position')
|
||||
|
||||
# --- EXIT CONDITIONS ---
|
||||
if position is not None:
|
||||
held_too_long = False
|
||||
if 'entry_idx' in position:
|
||||
candles_held = idx - position['entry_idx']
|
||||
held_too_long = candles_held >= g.max_hold_candles
|
||||
|
||||
pos_dir = position.get('direction', 'long')
|
||||
|
||||
if pos_dir == 'long':
|
||||
if rsi > g.rsi_overbought or held_too_long or macd_hist < 0:
|
||||
return {'action': 'sell'}
|
||||
elif pos_dir == 'short':
|
||||
if rsi < g.rsi_oversold or held_too_long or macd_hist > 0:
|
||||
return {'action': 'cover'}
|
||||
return {'action': 'hold'}
|
||||
|
||||
# --- LONG ENTRY (scalp) ---
|
||||
bullish_trend = current_price > fast_ma
|
||||
rsi_buy_zone = g.rsi_oversold < rsi < (g.rsi_overbought - 5)
|
||||
macd_bullish = macd_hist > 0
|
||||
volume_active = vol_ratio > g.volume_surge_threshold
|
||||
|
||||
# Bollinger bounce: price near lower band = mean reversion long
|
||||
bb_bounce_long = False
|
||||
if bb_lower is not None and idx < len(bb_lower):
|
||||
bb_bounce_long = current_price <= bb_lower[idx] * 1.005
|
||||
|
||||
long_signal = (bullish_trend and rsi_buy_zone and macd_bullish and volume_active) or \
|
||||
(bb_bounce_long and rsi < 35 and volume_active)
|
||||
|
||||
if long_signal:
|
||||
stop_loss = current_price - (atr * g.stop_loss_atr_mult)
|
||||
take_profit = current_price + (atr * g.take_profit_atr_mult)
|
||||
confidence = min(1.0, (vol_ratio - 1) * 0.4 + 0.3)
|
||||
if bb_bounce_long:
|
||||
confidence = min(1.0, confidence + 0.15)
|
||||
|
||||
return {
|
||||
'action': 'buy',
|
||||
'amount_pct': g.max_position_pct,
|
||||
'stop_loss': stop_loss,
|
||||
'take_profit': take_profit,
|
||||
'confidence': confidence,
|
||||
}
|
||||
|
||||
# --- SHORT ENTRY (scalp) ---
|
||||
bearish_trend = current_price < fast_ma
|
||||
rsi_sell_zone = (g.rsi_oversold + 5) < rsi < g.rsi_overbought
|
||||
macd_bearish = macd_hist < 0
|
||||
|
||||
# Bollinger rejection: price near upper band = mean reversion short
|
||||
bb_bounce_short = False
|
||||
if bb_upper is not None and idx < len(bb_upper):
|
||||
bb_bounce_short = current_price >= bb_upper[idx] * 0.995
|
||||
|
||||
short_signal = (bearish_trend and rsi_sell_zone and macd_bearish and volume_active) or \
|
||||
(bb_bounce_short and rsi > 65 and volume_active)
|
||||
|
||||
if short_signal:
|
||||
# For shorts: stop is ABOVE, take profit is BELOW
|
||||
short_stop = current_price + (atr * g.stop_loss_atr_mult)
|
||||
short_tp = current_price - (atr * g.take_profit_atr_mult)
|
||||
confidence = min(1.0, (vol_ratio - 1) * 0.4 + 0.3)
|
||||
if bb_bounce_short:
|
||||
confidence = min(1.0, confidence + 0.15)
|
||||
|
||||
return {
|
||||
'action': 'short',
|
||||
'amount_pct': g.max_position_pct,
|
||||
'stop_loss': short_stop,
|
||||
'take_profit': short_tp,
|
||||
'short_stop_loss': short_stop,
|
||||
'short_take_profit': short_tp,
|
||||
'confidence': confidence,
|
||||
}
|
||||
|
||||
return {'action': 'hold'}
|
||||
|
||||
return strategy_fn
|
||||
|
||||
|
||||
def genome_to_signals(genome, df: pd.DataFrame) -> Dict[str, np.ndarray]:
|
||||
"""
|
||||
Vectorized signal generation for fast backtesting (long + short scalping).
|
||||
Pre-computes all indicators and generates entry/exit signal arrays for both sides.
|
||||
|
||||
Returns dict with:
|
||||
'entry': boolean array (True = long entry signal)
|
||||
'exit': boolean array (True = long exit signal)
|
||||
'short_entry': boolean array (True = short entry signal)
|
||||
'short_exit': boolean array (True = short exit/cover signal)
|
||||
'stop_loss': float array (long SL price at each bar)
|
||||
'take_profit': float array (long TP price at each bar)
|
||||
'short_stop_loss': float array (short SL price, ABOVE entry)
|
||||
'short_take_profit': float array (short TP price, BELOW entry)
|
||||
'amount_pct': float (position size)
|
||||
'indicators': dict of pre-computed indicator arrays
|
||||
"""
|
||||
g = genome
|
||||
ind = _precompute_indicators(df, g)
|
||||
|
||||
min_idx = max(g.slow_ma_period, 30)
|
||||
|
||||
# --- LONG ENTRY ---
|
||||
bullish_trend = ind['close'] > ind['fast_ma']
|
||||
rsi_buy_zone = (ind['rsi'] > g.rsi_oversold) & (ind['rsi'] < (g.rsi_overbought - 5))
|
||||
macd_bullish = ind['macd_hist'] > 0
|
||||
volume_active = ind['vol_ratio'] > g.volume_surge_threshold
|
||||
|
||||
# BB bounce long
|
||||
bb_bounce_long = np.zeros(len(ind['close']), dtype=bool)
|
||||
if 'bb_lower' in ind:
|
||||
bb_bounce_long = ind['close'] <= ind['bb_lower'] * 1.005
|
||||
bb_long_entry = bb_bounce_long & (ind['rsi'] < 35) & volume_active
|
||||
|
||||
entry = (bullish_trend & rsi_buy_zone & macd_bullish & volume_active) | bb_long_entry
|
||||
entry[:min_idx] = False
|
||||
|
||||
# Long exit: RSI overbought or MACD turns bearish
|
||||
exit_signal = (ind['rsi'] > g.rsi_overbought) | (ind['macd_hist'] < 0)
|
||||
exit_signal[:min_idx] = False
|
||||
|
||||
# --- SHORT ENTRY ---
|
||||
bearish_trend = ind['close'] < ind['fast_ma']
|
||||
rsi_sell_zone = (ind['rsi'] > (g.rsi_oversold + 5)) & (ind['rsi'] < g.rsi_overbought)
|
||||
macd_bearish = ind['macd_hist'] < 0
|
||||
|
||||
# BB bounce short
|
||||
bb_bounce_short = np.zeros(len(ind['close']), dtype=bool)
|
||||
if 'bb_upper' in ind:
|
||||
bb_bounce_short = ind['close'] >= ind['bb_upper'] * 0.995
|
||||
bb_short_entry = bb_bounce_short & (ind['rsi'] > 65) & volume_active
|
||||
|
||||
short_entry = (bearish_trend & rsi_sell_zone & macd_bearish & volume_active) | bb_short_entry
|
||||
short_entry[:min_idx] = False
|
||||
|
||||
# Short exit: RSI oversold or MACD turns bullish
|
||||
short_exit = (ind['rsi'] < g.rsi_oversold) | (ind['macd_hist'] > 0)
|
||||
short_exit[:min_idx] = False
|
||||
|
||||
# SL/TP levels (long)
|
||||
stop_loss = ind['close'] - (ind['atr'] * g.stop_loss_atr_mult)
|
||||
take_profit = ind['close'] + (ind['atr'] * g.take_profit_atr_mult)
|
||||
|
||||
# SL/TP levels (short - inverted)
|
||||
short_stop_loss = ind['close'] + (ind['atr'] * g.stop_loss_atr_mult)
|
||||
short_take_profit = ind['close'] - (ind['atr'] * g.take_profit_atr_mult)
|
||||
|
||||
return {
|
||||
'entry': entry,
|
||||
'exit': exit_signal,
|
||||
'short_entry': short_entry,
|
||||
'short_exit': short_exit,
|
||||
'stop_loss': stop_loss,
|
||||
'take_profit': take_profit,
|
||||
'short_stop_loss': short_stop_loss,
|
||||
'short_take_profit': short_take_profit,
|
||||
'amount_pct': g.max_position_pct,
|
||||
'max_hold_candles': g.max_hold_candles,
|
||||
'indicators': ind,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_genome_signal(genome, df: pd.DataFrame, idx: int) -> Dict:
|
||||
"""
|
||||
Evaluate a genome's strategy at a specific index.
|
||||
Returns signal dict with action, confidence, stop_loss, take_profit.
|
||||
Used by the RL agent to get the GA signal component.
|
||||
Supports long, short, and hold signals.
|
||||
"""
|
||||
if idx < max(genome.slow_ma_period, 30) or idx >= len(df):
|
||||
return {'signal': 'hold', 'confidence': 0.0}
|
||||
|
||||
strategy_fn = genome_to_strategy(genome)
|
||||
state = {'position': None, 'capital': 1000, 'num_trades': 0}
|
||||
result = strategy_fn(state, idx, df, genome.to_dict())
|
||||
|
||||
action = result.get('action', 'hold')
|
||||
# Normalize: 'short' action -> 'sell' signal for RL
|
||||
signal = action
|
||||
if action == 'short':
|
||||
signal = 'sell'
|
||||
elif action == 'cover':
|
||||
signal = 'buy'
|
||||
|
||||
return {
|
||||
'signal': signal,
|
||||
'confidence': result.get('confidence', 0.0),
|
||||
'stop_loss': result.get('stop_loss'),
|
||||
'take_profit': result.get('take_profit'),
|
||||
'short_stop_loss': result.get('short_stop_loss'),
|
||||
'short_take_profit': result.get('short_take_profit'),
|
||||
'position_pct': result.get('amount_pct', 0.1),
|
||||
}
|
||||
|
||||
|
||||
# --- Pre-computation helpers ---
|
||||
|
||||
def _precompute_indicators(df: pd.DataFrame, genome) -> Dict[str, np.ndarray]:
|
||||
"""Pre-compute all indicators as numpy arrays in one pass."""
|
||||
close = df['close'].values.astype(np.float64)
|
||||
high = df['high'].values.astype(np.float64)
|
||||
low = df['low'].values.astype(np.float64)
|
||||
volume = df['volume'].values.astype(np.float64)
|
||||
|
||||
# Moving averages (simple cumsum trick)
|
||||
fast_ma = _rolling_mean(close, genome.fast_ma_period)
|
||||
slow_ma = _rolling_mean(close, genome.slow_ma_period)
|
||||
|
||||
# RSI
|
||||
rsi = _compute_rsi_array(close, genome.rsi_period)
|
||||
|
||||
# MACD
|
||||
ema_fast = _ema_array(close, genome.macd_fast)
|
||||
ema_slow = _ema_array(close, genome.macd_slow)
|
||||
macd_line = ema_fast - ema_slow
|
||||
signal_line = _ema_array(macd_line, genome.macd_signal)
|
||||
macd_hist = macd_line - signal_line
|
||||
|
||||
# ATR
|
||||
atr = _compute_atr_array(high, low, close, genome.atr_period)
|
||||
|
||||
# Volume ratio (current volume / 20-period SMA of volume)
|
||||
vol_sma = _rolling_mean(volume, 20)
|
||||
vol_ratio = np.where(vol_sma > 0, volume / vol_sma, 1.0)
|
||||
|
||||
# Bollinger Bands for mean-reversion scalping
|
||||
bb_mid = _rolling_mean(close, genome.bb_period)
|
||||
bb_std = np.full(len(close), 0.0)
|
||||
for i in range(genome.bb_period - 1, len(close)):
|
||||
bb_std[i] = np.std(close[max(0, i - genome.bb_period + 1):i + 1])
|
||||
for i in range(genome.bb_period - 1):
|
||||
bb_std[i] = np.std(close[:i + 1]) if i > 0 else 0.0
|
||||
bb_upper = bb_mid + genome.bb_std * bb_std
|
||||
bb_lower = bb_mid - genome.bb_std * bb_std
|
||||
|
||||
return {
|
||||
'close': close,
|
||||
'high': high,
|
||||
'low': low,
|
||||
'volume': volume,
|
||||
'fast_ma': fast_ma,
|
||||
'slow_ma': slow_ma,
|
||||
'rsi': rsi,
|
||||
'macd_line': macd_line,
|
||||
'macd_signal': signal_line,
|
||||
'macd_hist': macd_hist,
|
||||
'atr': atr,
|
||||
'vol_ratio': vol_ratio,
|
||||
'bb_upper': bb_upper,
|
||||
'bb_mid': bb_mid,
|
||||
'bb_lower': bb_lower,
|
||||
}
|
||||
|
||||
|
||||
def _rolling_mean(arr: np.ndarray, period: int) -> np.ndarray:
|
||||
"""Fast rolling mean using cumsum."""
|
||||
n = len(arr)
|
||||
result = np.full(n, np.nan)
|
||||
if n < period:
|
||||
for i in range(n):
|
||||
result[i] = np.mean(arr[:i + 1])
|
||||
return result
|
||||
cs = np.cumsum(arr)
|
||||
result[period - 1:] = (cs[period - 1:] - np.concatenate([[0], cs[:n - period]])) / period
|
||||
# Fill initial values with expanding mean
|
||||
for i in range(period - 1):
|
||||
result[i] = np.mean(arr[:i + 1])
|
||||
return result
|
||||
|
||||
|
||||
def _ema_array(arr: np.ndarray, period: int) -> np.ndarray:
|
||||
"""Compute EMA for entire array in one pass."""
|
||||
n = len(arr)
|
||||
result = np.empty(n)
|
||||
multiplier = 2.0 / (period + 1)
|
||||
result[0] = arr[0]
|
||||
for i in range(1, n):
|
||||
result[i] = (arr[i] - result[i - 1]) * multiplier + result[i - 1]
|
||||
return result
|
||||
|
||||
|
||||
def _compute_rsi_array(close: np.ndarray, period: int) -> np.ndarray:
|
||||
"""Compute RSI for entire array using Wilder's smoothing."""
|
||||
n = len(close)
|
||||
rsi = np.full(n, 50.0)
|
||||
if n < period + 1:
|
||||
return rsi
|
||||
|
||||
deltas = np.diff(close)
|
||||
gains = np.where(deltas > 0, deltas, 0.0)
|
||||
losses = np.where(deltas < 0, -deltas, 0.0)
|
||||
|
||||
# Initial averages (SMA)
|
||||
avg_gain = np.mean(gains[:period])
|
||||
avg_loss = np.mean(losses[:period])
|
||||
|
||||
if avg_loss > 0:
|
||||
rs = avg_gain / avg_loss
|
||||
rsi[period] = 100.0 - (100.0 / (1.0 + rs))
|
||||
else:
|
||||
rsi[period] = 100.0
|
||||
|
||||
# Wilder's smoothing for remaining
|
||||
for i in range(period, len(deltas)):
|
||||
avg_gain = (avg_gain * (period - 1) + gains[i]) / period
|
||||
avg_loss = (avg_loss * (period - 1) + losses[i]) / period
|
||||
if avg_loss > 0:
|
||||
rs = avg_gain / avg_loss
|
||||
rsi[i + 1] = 100.0 - (100.0 / (1.0 + rs))
|
||||
else:
|
||||
rsi[i + 1] = 100.0
|
||||
|
||||
return rsi
|
||||
|
||||
|
||||
def _compute_atr_array(high: np.ndarray, low: np.ndarray,
|
||||
close: np.ndarray, period: int) -> np.ndarray:
|
||||
"""Compute ATR for entire array using Wilder's smoothing."""
|
||||
n = len(close)
|
||||
atr = np.full(n, 0.0)
|
||||
if n < 2:
|
||||
return atr
|
||||
|
||||
# True range
|
||||
tr = np.empty(n)
|
||||
tr[0] = high[0] - low[0]
|
||||
for i in range(1, n):
|
||||
tr[i] = max(high[i] - low[i],
|
||||
abs(high[i] - close[i - 1]),
|
||||
abs(low[i] - close[i - 1]))
|
||||
|
||||
# Initial ATR = SMA of first `period` TRs
|
||||
if n >= period:
|
||||
atr[period - 1] = np.mean(tr[:period])
|
||||
# Wilder's smoothing
|
||||
for i in range(period, n):
|
||||
atr[i] = (atr[i - 1] * (period - 1) + tr[i]) / period
|
||||
# Fill early values with expanding mean
|
||||
for i in range(period - 1):
|
||||
atr[i] = np.mean(tr[:i + 1])
|
||||
else:
|
||||
for i in range(n):
|
||||
atr[i] = np.mean(tr[:i + 1])
|
||||
|
||||
return atr
|
||||
+384
-52
@@ -1,11 +1,23 @@
|
||||
"""
|
||||
Trade Executor
|
||||
Executes live trades via Alpaca broker with position management.
|
||||
|
||||
ENHANCED with:
|
||||
- Proper correlation-based position limits (using max_correlated_positions config)
|
||||
- Drawdown-scaled position sizing
|
||||
- Enforced stop losses with tighter defaults
|
||||
- Active position reduction on circuit breaker / drawdown stop
|
||||
- Per-sector position limits
|
||||
- HARD maximum SHORT position cap (broker-level enforcement)
|
||||
- Per-position stop loss defaults (ATR-based, always set)
|
||||
- Broker-position sync check (authoritative position counting)
|
||||
- Yesterday-close drawdown circuit breaker (faster response)
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from loguru import logger
|
||||
import numpy as np
|
||||
|
||||
|
||||
class TradingExecutor:
|
||||
@@ -28,6 +40,109 @@ class TradingExecutor:
|
||||
self.config = config
|
||||
self.commission_rate = config.get('commission_rate', 0.001)
|
||||
|
||||
# Stop loss / take profit defaults (can be overridden by strategy)
|
||||
self.default_stop_loss_pct = config.get('stop_loss_pct', 1.0) / 100
|
||||
self.default_take_profit_pct = config.get('take_profit_pct', 1.5) / 100
|
||||
|
||||
# Track recent prices for volatility calculation
|
||||
self.recent_returns: Dict[str, List[float]] = {}
|
||||
|
||||
# Sector map for correlation
|
||||
self.sector_map = {
|
||||
'SPY': 'broad_market', 'QQQ': 'tech', 'IWM': 'small_cap', 'DIA': 'blue_chip',
|
||||
'AAPL': 'tech', 'MSFT': 'tech', 'GOOGL': 'tech', 'AMZN': 'tech', 'NVDA': 'tech', 'META': 'tech',
|
||||
'TSLA': 'auto/energy',
|
||||
'JPM': 'financials', 'BAC': 'financials', 'GS': 'financials', 'MS': 'financials',
|
||||
'JNJ': 'healthcare', 'UNH': 'healthcare',
|
||||
'WMT': 'consumer', 'PG': 'consumer',
|
||||
'XLE': 'energy', 'CVX': 'energy', 'XOM': 'energy',
|
||||
'LMT': 'defense', 'RTX': 'defense', 'NOC': 'defense', 'GD': 'defense',
|
||||
'USO': 'commodities', 'GLD': 'metals',
|
||||
}
|
||||
|
||||
def _get_sector(self, symbol: str) -> str:
|
||||
return self.sector_map.get(symbol, 'other')
|
||||
|
||||
def _update_volatility(self, symbol: str, current_price: float):
|
||||
"""Track price returns for volatility calculation"""
|
||||
if symbol not in self.recent_returns:
|
||||
self.recent_returns[symbol] = []
|
||||
# Returns are tracked externally; just store prices
|
||||
# Volatility is computed by safety manager
|
||||
|
||||
def _apply_drawdown_scaling(self, invest_amount: float, portfolio_value: float) -> float:
|
||||
"""Scale down position size based on current drawdown"""
|
||||
scale = self.safety.get_position_scale(portfolio_value)
|
||||
vol_mult = self.safety.current_volatility_mult
|
||||
return invest_amount * scale * vol_mult
|
||||
|
||||
def _check_correlation_limits(self, symbol: str, direction: str,
|
||||
open_positions: List[Dict]) -> Tuple[bool, str]:
|
||||
"""
|
||||
Check if adding this position would violate correlation limits.
|
||||
|
||||
Returns: (allowed: bool, reason: str)
|
||||
"""
|
||||
max_corr = self.safety.max_correlated_positions
|
||||
max_same_dir = self.safety.max_same_direction
|
||||
|
||||
# Count current positions in same direction
|
||||
same_dir_positions = [
|
||||
p for p in open_positions
|
||||
if p.get('metadata', {}).get('direction') == direction
|
||||
]
|
||||
|
||||
# Check same-direction limit
|
||||
if len(same_dir_positions) >= max_same_dir:
|
||||
return False, (f"Max {max_same_dir} {direction} positions reached "
|
||||
f"(have {len(same_dir_positions)})")
|
||||
|
||||
# Check sector correlation limit
|
||||
sector = self._get_sector(symbol)
|
||||
if sector != 'other' and sector != 'broad_market':
|
||||
same_sector_same_dir = [
|
||||
p for p in same_dir_positions
|
||||
if self._get_sector(p.get('symbol', '')) == sector
|
||||
]
|
||||
if len(same_sector_same_dir) >= max_corr:
|
||||
return False, (f"Max {max_corr} {direction} positions in "
|
||||
f"{sector} sector (have {len(same_sector_same_dir)})")
|
||||
|
||||
# Also limit broad market ETF correlation (SPY, QQQ, IWM, DIA all correlate)
|
||||
if symbol in ('SPY', 'QQQ', 'IWM', 'DIA'):
|
||||
market_etfs = [p for p in same_dir_positions if p.get('symbol') in ('SPY', 'QQQ', 'IWM', 'DIA')]
|
||||
if len(market_etfs) >= 2:
|
||||
return False, f"Max 2 market ETFs as {direction} positions (have {len(market_etfs)})"
|
||||
|
||||
return True, "OK"
|
||||
|
||||
def _enforce_stop_loss_tightness(self, stop_loss: float, entry_price: float,
|
||||
direction: str) -> float:
|
||||
"""
|
||||
Ensure stop loss is tight enough - don't let wide stops blow up risk.
|
||||
If strategy provides too-wide stop, override with our default.
|
||||
"""
|
||||
if stop_loss is None:
|
||||
return None
|
||||
|
||||
if direction == 'short':
|
||||
# Short stop: price goes UP to hit stop (bad)
|
||||
stop_distance_pct = (stop_loss - entry_price) / entry_price
|
||||
else:
|
||||
# Long stop: price goes DOWN to hit stop (bad)
|
||||
stop_distance_pct = (entry_price - stop_loss) / entry_price
|
||||
|
||||
# If strategy stop is wider than 3x our default, use our default instead
|
||||
if stop_distance_pct > self.default_stop_loss_pct * 3:
|
||||
logger.warning(f"Stop loss {stop_distance_pct:.2%} too wide, "
|
||||
f"using default {self.default_stop_loss_pct:.2%}")
|
||||
if direction == 'short':
|
||||
return entry_price * (1 + self.default_stop_loss_pct)
|
||||
else:
|
||||
return entry_price * (1 - self.default_stop_loss_pct)
|
||||
|
||||
return stop_loss
|
||||
|
||||
def execute_signal(self, symbol: str, action: int, current_price: float,
|
||||
strategy_params: Dict = None, combined_equity: float = None) -> Optional[Dict]:
|
||||
"""
|
||||
@@ -71,16 +186,41 @@ class TradingExecutor:
|
||||
# Update safety peak
|
||||
self.safety.update_peak_equity(safety_equity)
|
||||
|
||||
# Get current positions
|
||||
# Get current open positions from STORE (known trades)
|
||||
open_positions = self.store.get_open_positions()
|
||||
open_for_symbol = [p for p in open_positions if p['symbol'] == symbol]
|
||||
num_positions = len(set(p['symbol'] for p in open_positions))
|
||||
|
||||
# DIRECTIONAL DIVERSITY: Don't let all positions be the same direction
|
||||
all_directions = [p.get('metadata', {}).get('direction', 'long') for p in open_positions]
|
||||
num_long = sum(1 for d in all_directions if d == 'long')
|
||||
num_short = sum(1 for d in all_directions if d == 'short')
|
||||
max_one_direction = max(num_positions - 1, 3) # At least allow 3 of same direction
|
||||
# CRITICAL: Get ACTUAL broker positions (authoritative) for hard cap checks
|
||||
# Store only knows what WE opened — broker knows everything (including manual trades)
|
||||
try:
|
||||
broker_positions = self.broker.get_positions()
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not get broker positions for hard cap check: {e}")
|
||||
broker_positions = []
|
||||
|
||||
# HARD SHORT CAP CHECK: If broker already has max_short_positions shorts, block more
|
||||
# This is the PRIMARY defense against the "13 shorts" scenario
|
||||
broker_short_count = sum(
|
||||
1 for p in broker_positions
|
||||
if p.get('side') == 'sell_short' or p.get('qty', 0) < 0
|
||||
)
|
||||
if action in (5, 6) and broker_short_count >= self.safety.max_short_positions:
|
||||
logger.warning(
|
||||
f"HARD CAP: Broker has {broker_short_count} shorts "
|
||||
f"(max={self.safety.max_short_positions}). Short blocked."
|
||||
)
|
||||
return None
|
||||
|
||||
# BROKER POSITION SYNC WARNING
|
||||
store_short_count = sum(
|
||||
1 for p in open_positions
|
||||
if p.get('metadata', {}).get('direction') == 'short'
|
||||
)
|
||||
if broker_short_count != store_short_count:
|
||||
logger.warning(
|
||||
f"BROKER/STORE SYNC MISMATCH: Broker={broker_short_count} shorts, "
|
||||
f"Store={store_short_count} shorts. Using broker as authoritative."
|
||||
)
|
||||
|
||||
# BUY LONG actions
|
||||
if action in (1, 2):
|
||||
@@ -88,33 +228,42 @@ class TradingExecutor:
|
||||
logger.debug(f"Already have position in {symbol}, skipping buy")
|
||||
return None
|
||||
|
||||
# Block if too many longs already
|
||||
if num_long >= max_one_direction:
|
||||
logger.debug(f"Too many longs ({num_long}), blocking new long on {symbol}")
|
||||
# CORRELATION CHECK: Don't pile into same direction (uses store positions)
|
||||
allowed, reason = self._check_correlation_limits(symbol, 'long', open_positions)
|
||||
if not allowed:
|
||||
logger.debug(f"Correlation blocked LONG {symbol}: {reason}")
|
||||
return None
|
||||
|
||||
pct = 0.25 if action == 1 else 0.50
|
||||
invest = cash * pct
|
||||
|
||||
max_position_pct = self.config.get('max_position_pct', 12) / 100
|
||||
max_invest = equity * max_position_pct
|
||||
# Apply drawdown + volatility scaling
|
||||
invest = self._apply_drawdown_scaling(invest, safety_equity)
|
||||
|
||||
max_position_pct = self.config.get('max_position_pct', 8) / 100
|
||||
effective_max = max_position_pct * self.safety.get_position_scale(safety_equity)
|
||||
max_invest = safety_equity * effective_max
|
||||
if invest > max_invest:
|
||||
invest = max_invest
|
||||
|
||||
if invest < self.config.get('min_trade_value', 3):
|
||||
if invest < self.config.get('min_trade_value', 50):
|
||||
logger.debug(f"Invest amount ${invest:.2f} below minimum")
|
||||
return None
|
||||
|
||||
shares = int(invest / current_price)
|
||||
if shares < 1:
|
||||
# Try fractional
|
||||
shares = round(invest / current_price, 4)
|
||||
if shares * current_price < self.config.get('min_trade_value', 3):
|
||||
if shares * current_price < self.config.get('min_trade_value', 50):
|
||||
logger.debug(f"Share value ${shares * current_price:.2f} below minimum")
|
||||
return None
|
||||
|
||||
allowed, reason = self.safety.validate_trade(
|
||||
symbol, 'buy', shares, current_price, safety_equity, num_positions
|
||||
symbol, 'buy', shares, current_price, safety_equity, open_positions,
|
||||
broker_positions=broker_positions
|
||||
)
|
||||
if not allowed:
|
||||
logger.debug(f"Trade blocked: {reason}")
|
||||
logger.debug(f"Trade blocked by safety: {reason}")
|
||||
return None
|
||||
|
||||
try:
|
||||
@@ -124,9 +273,31 @@ class TradingExecutor:
|
||||
logger.error(f"Error placing buy order: {e}")
|
||||
return None
|
||||
|
||||
# Stop loss and take profit with tightness enforcement + defaults
|
||||
stop_loss = strategy_params.get('stop_loss')
|
||||
take_profit = strategy_params.get('take_profit')
|
||||
|
||||
# CRITICAL: If GA didn't provide stop loss, use SAFETY MANAGER DEFAULTS
|
||||
# This ensures EVERY position has a stop loss — the #1 defense against blowup
|
||||
if stop_loss is None:
|
||||
stop_loss = self.safety.get_default_stop_loss(current_price, 'long')
|
||||
logger.info(f"Applying DEFAULT stop loss for LONG {symbol}: ${stop_loss:.4f} "
|
||||
f"({((current_price - stop_loss) / current_price) * 100:.2f}% from entry)")
|
||||
|
||||
# Enforce tightness ceiling on the stop
|
||||
stop_loss = self._enforce_stop_loss_tightness(stop_loss, current_price, 'long')
|
||||
|
||||
# CRITICAL: If GA didn't provide take profit, use SAFETY MANAGER DEFAULTS
|
||||
if take_profit is None or take_profit == current_price:
|
||||
take_profit = self.safety.get_default_take_profit(current_price, 'long')
|
||||
logger.info(f"Applying DEFAULT take profit for LONG {symbol}: ${take_profit:.4f}")
|
||||
|
||||
if take_profit and take_profit != current_price:
|
||||
tp_distance = (take_profit - current_price) / current_price
|
||||
if tp_distance < self.default_take_profit_pct * 0.5:
|
||||
# TP too tight relative to default
|
||||
take_profit = current_price * (1 + self.default_take_profit_pct)
|
||||
|
||||
trade = {
|
||||
'symbol': symbol,
|
||||
'side': 'buy',
|
||||
@@ -143,6 +314,7 @@ class TradingExecutor:
|
||||
'invest_pct': pct,
|
||||
'equity_at_entry': equity,
|
||||
'direction': 'long',
|
||||
'sector': self._get_sector(symbol),
|
||||
},
|
||||
}
|
||||
trade_id = self.store.record_trade(trade)
|
||||
@@ -164,10 +336,9 @@ class TradingExecutor:
|
||||
close_shares = abs(total_shares)
|
||||
|
||||
close_shares = round(close_shares, 4)
|
||||
if close_shares * current_price < self.config.get('min_trade_value', 1):
|
||||
if close_shares * current_price < self.config.get('min_trade_value', 50):
|
||||
close_shares = abs(total_shares)
|
||||
|
||||
# Determine order side (opposite of position direction)
|
||||
close_side = 'buy' if is_short else 'sell'
|
||||
|
||||
try:
|
||||
@@ -201,27 +372,12 @@ class TradingExecutor:
|
||||
'direction': 'short' if is_short else 'long',
|
||||
}
|
||||
else:
|
||||
self.safety.record_trade_result(pnl)
|
||||
self.store.close_position(
|
||||
position['id'], current_price, datetime.utcnow(),
|
||||
self.safety.record_trade_result(pnl * (close_shares / abs(total_shares)))
|
||||
self.store.reduce_position(
|
||||
position['id'], close_shares, current_price, datetime.utcnow(),
|
||||
fees=close_shares * current_price * self.commission_rate
|
||||
)
|
||||
|
||||
remaining = abs(total_shares) - close_shares
|
||||
if remaining > 0:
|
||||
self.store.record_trade({
|
||||
'symbol': symbol,
|
||||
'side': position['side'],
|
||||
'amount': remaining,
|
||||
'entry_price': position['entry_price'],
|
||||
'entry_time': position['entry_time'],
|
||||
'strategy_id': position.get('strategy_id', 'auto'),
|
||||
'stop_loss': position.get('stop_loss'),
|
||||
'take_profit': position.get('take_profit'),
|
||||
'status': 'open',
|
||||
'metadata': position.get('metadata', {}),
|
||||
})
|
||||
|
||||
return {
|
||||
'symbol': symbol,
|
||||
'side': close_side,
|
||||
@@ -230,6 +386,7 @@ class TradingExecutor:
|
||||
'pnl': round(pnl, 2),
|
||||
'action': action,
|
||||
'direction': 'short' if is_short else 'long',
|
||||
'reduced': True,
|
||||
}
|
||||
|
||||
# SHORT actions
|
||||
@@ -238,33 +395,41 @@ class TradingExecutor:
|
||||
logger.debug(f"Already have position in {symbol}, skipping short")
|
||||
return None
|
||||
|
||||
# Block if too many shorts already
|
||||
if num_short >= max_one_direction:
|
||||
logger.debug(f"Too many shorts ({num_short}), blocking new short on {symbol}")
|
||||
# CORRELATION CHECK
|
||||
allowed, reason = self._check_correlation_limits(symbol, 'short', open_positions)
|
||||
if not allowed:
|
||||
logger.debug(f"Correlation blocked SHORT {symbol}: {reason}")
|
||||
return None
|
||||
|
||||
pct = 0.25 if action == 5 else 0.50
|
||||
invest = cash * pct
|
||||
|
||||
max_position_pct = self.config.get('max_position_pct', 12) / 100
|
||||
max_invest = equity * max_position_pct
|
||||
# Apply drawdown + volatility scaling
|
||||
invest = self._apply_drawdown_scaling(invest, safety_equity)
|
||||
|
||||
max_position_pct = self.config.get('max_position_pct', 8) / 100
|
||||
effective_max = max_position_pct * self.safety.get_position_scale(safety_equity)
|
||||
max_invest = safety_equity * effective_max
|
||||
if invest > max_invest:
|
||||
invest = max_invest
|
||||
|
||||
if invest < self.config.get('min_trade_value', 3):
|
||||
if invest < self.config.get('min_trade_value', 50):
|
||||
logger.debug(f"Short invest amount ${invest:.2f} below minimum")
|
||||
return None
|
||||
|
||||
shares = int(invest / current_price)
|
||||
if shares < 1:
|
||||
shares = round(invest / current_price, 4)
|
||||
if shares * current_price < self.config.get('min_trade_value', 3):
|
||||
if shares * current_price < self.config.get('min_trade_value', 50):
|
||||
logger.debug(f"Short share value ${shares * current_price:.2f} below minimum")
|
||||
return None
|
||||
|
||||
allowed, reason = self.safety.validate_trade(
|
||||
symbol, 'sell', shares, current_price, safety_equity, num_positions
|
||||
symbol, 'sell', shares, current_price, safety_equity, open_positions,
|
||||
broker_positions=broker_positions
|
||||
)
|
||||
if not allowed:
|
||||
logger.debug(f"Short blocked: {reason}")
|
||||
logger.debug(f"Short blocked by safety: {reason}")
|
||||
return None
|
||||
|
||||
try:
|
||||
@@ -274,10 +439,24 @@ class TradingExecutor:
|
||||
logger.error(f"Error placing short order: {e}")
|
||||
return None
|
||||
|
||||
# For shorts, stop_loss is ABOVE entry, take_profit is BELOW
|
||||
# For shorts: stop_loss is ABOVE entry (price rises = bad), take_profit is BELOW
|
||||
stop_loss = strategy_params.get('short_stop_loss') or strategy_params.get('stop_loss')
|
||||
take_profit = strategy_params.get('short_take_profit') or strategy_params.get('take_profit')
|
||||
|
||||
# CRITICAL: If GA didn't provide stop loss, use SAFETY MANAGER DEFAULTS
|
||||
if stop_loss is None:
|
||||
stop_loss = self.safety.get_default_stop_loss(current_price, 'short')
|
||||
logger.info(f"Applying DEFAULT stop loss for SHORT {symbol}: ${stop_loss:.4f} "
|
||||
f"({((stop_loss - current_price) / current_price) * 100:.2f}% from entry)")
|
||||
|
||||
# Enforce tightness ceiling on the stop
|
||||
stop_loss = self._enforce_stop_loss_tightness(stop_loss, current_price, 'short')
|
||||
|
||||
# CRITICAL: If GA didn't provide take profit, use SAFETY MANAGER DEFAULTS
|
||||
if take_profit is None:
|
||||
take_profit = self.safety.get_default_take_profit(current_price, 'short')
|
||||
logger.info(f"Applying DEFAULT take profit for SHORT {symbol}: ${take_profit:.4f}")
|
||||
|
||||
trade = {
|
||||
'symbol': symbol,
|
||||
'side': 'sell',
|
||||
@@ -294,6 +473,7 @@ class TradingExecutor:
|
||||
'invest_pct': pct,
|
||||
'equity_at_entry': equity,
|
||||
'direction': 'short',
|
||||
'sector': self._get_sector(symbol),
|
||||
},
|
||||
}
|
||||
trade_id = self.store.record_trade(trade)
|
||||
@@ -305,12 +485,60 @@ class TradingExecutor:
|
||||
def check_exits(self, current_prices: Dict[str, float]) -> List[Dict]:
|
||||
"""
|
||||
Check all open positions for stop loss / take profit exits.
|
||||
Handles both long and short positions.
|
||||
Also checks for drawdown stop and forces partial closes if needed.
|
||||
Returns list of closed trades.
|
||||
"""
|
||||
closed = []
|
||||
open_positions = self.store.get_open_positions()
|
||||
|
||||
# Get combined portfolio for drawdown check
|
||||
try:
|
||||
portfolio = self.broker.get_portfolio()
|
||||
equity = portfolio['equity']
|
||||
except Exception:
|
||||
equity = 0
|
||||
|
||||
# Update yesterday's equity reference at start of each day
|
||||
self.safety.update_yesterday_equity(equity)
|
||||
|
||||
# Check if drawdown stop is triggered (peak-equity based, slow response)
|
||||
should_reduce_dd, dd_reason, reduce_pct = False, "", 0.0
|
||||
if equity > 0 and open_positions:
|
||||
should_reduce_dd, dd_reason, reduce_pct = \
|
||||
self.safety.check_drawdown_stop(equity, open_positions)
|
||||
|
||||
# NEW: Check YESTERDAY-CLOSE drawdown (FASTER response, catches intraday blowups)
|
||||
should_reduce_yc, yc_reason, yc_reduce_pct = False, "", 0.0
|
||||
if equity > 0 and open_positions:
|
||||
should_reduce_yc, yc_reason, yc_reduce_pct = \
|
||||
self.safety.check_yesterday_close_drawdown(equity)
|
||||
# Use whichever is more urgent
|
||||
if should_reduce_yc and not should_reduce_dd:
|
||||
should_reduce_dd = True
|
||||
dd_reason = yc_reason
|
||||
reduce_pct = yc_reduce_pct
|
||||
logger.warning(f"YESTERDAY-CLOSE CIRCUIT BREAKER TRIGGERED: {yc_reason}")
|
||||
|
||||
# CRITICAL: If we have 10+ positions all losing, force reduce regardless of drawdown
|
||||
# This catches the "13 shorts all losing" scenario before drawdown thresholds are hit
|
||||
if len(open_positions) >= 10:
|
||||
losing_count = 0
|
||||
for pos in open_positions:
|
||||
sym = pos.get('symbol')
|
||||
px = current_prices.get(sym)
|
||||
if px and pos.get('entry_price'):
|
||||
is_short = pos.get('metadata', {}).get('direction') == 'short'
|
||||
if is_short and px > pos['entry_price']:
|
||||
losing_count += 1
|
||||
elif not is_short and px < pos['entry_price']:
|
||||
losing_count += 1
|
||||
if losing_count >= 8:
|
||||
logger.critical(
|
||||
f"CRISIS MODE: {losing_count}/{len(open_positions)} positions losing money! "
|
||||
f"Force-reducing 50% of all positions immediately."
|
||||
)
|
||||
return self.force_reduce_all_positions(0.50, current_prices)
|
||||
|
||||
for position in open_positions:
|
||||
symbol = position['symbol']
|
||||
price = current_prices.get(symbol)
|
||||
@@ -319,10 +547,13 @@ class TradingExecutor:
|
||||
|
||||
should_exit = False
|
||||
exit_reason = ''
|
||||
should_reduce = False
|
||||
reduce_amount = 0
|
||||
|
||||
is_short = position.get('metadata', {}).get('direction') == 'short'
|
||||
|
||||
# Normal stop loss / take profit checks
|
||||
if is_short:
|
||||
# Short position: stop_loss is ABOVE entry, take_profit is BELOW
|
||||
if position.get('stop_loss') and price >= position['stop_loss']:
|
||||
should_exit = True
|
||||
exit_reason = 'stop_loss'
|
||||
@@ -330,7 +561,6 @@ class TradingExecutor:
|
||||
should_exit = True
|
||||
exit_reason = 'take_profit'
|
||||
else:
|
||||
# Long position: stop_loss is BELOW entry, take_profit is ABOVE
|
||||
if position.get('stop_loss') and price <= position['stop_loss']:
|
||||
should_exit = True
|
||||
exit_reason = 'stop_loss'
|
||||
@@ -338,8 +568,13 @@ class TradingExecutor:
|
||||
should_exit = True
|
||||
exit_reason = 'take_profit'
|
||||
|
||||
# Drawdown stop: force partial close
|
||||
if not should_exit and should_reduce_dd:
|
||||
should_reduce = True
|
||||
reduce_amount = abs(position['amount']) * reduce_pct
|
||||
exit_reason = f'drawdown_stop'
|
||||
|
||||
if should_exit:
|
||||
# Close side is opposite of position direction
|
||||
close_side = 'buy' if is_short else 'sell'
|
||||
try:
|
||||
self.broker.place_market_order(
|
||||
@@ -373,6 +608,96 @@ class TradingExecutor:
|
||||
'direction': 'short' if is_short else 'long',
|
||||
})
|
||||
|
||||
elif should_reduce and reduce_amount > 0:
|
||||
# Partial close due to drawdown stop
|
||||
reduce_amount = round(reduce_amount, 4)
|
||||
if reduce_amount < 0.0001:
|
||||
continue
|
||||
|
||||
close_side = 'buy' if is_short else 'sell'
|
||||
try:
|
||||
self.broker.place_market_order(
|
||||
symbol, reduce_amount, close_side
|
||||
)
|
||||
logger.warning(f"DRAWDOWN REDUCE ({reduce_amount:.2f} of {position['amount']:.2f}) "
|
||||
f"{symbol} @ ${price:.4f}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error reducing position for {symbol}: {e}")
|
||||
continue
|
||||
|
||||
if is_short:
|
||||
pnl = (position['entry_price'] - price) * reduce_amount
|
||||
else:
|
||||
pnl = (price - position['entry_price']) * reduce_amount
|
||||
|
||||
self.store.reduce_position(
|
||||
position['id'], reduce_amount, price, datetime.utcnow(),
|
||||
fees=reduce_amount * price * self.commission_rate
|
||||
)
|
||||
self.safety.record_trade_result(pnl * (reduce_amount / abs(position['amount'])))
|
||||
|
||||
closed.append({
|
||||
'symbol': symbol,
|
||||
'side': close_side,
|
||||
'amount': reduce_amount,
|
||||
'entry_price': position['entry_price'],
|
||||
'exit_price': price,
|
||||
'pnl': round(pnl, 2),
|
||||
'exit_reason': 'drawdown_reduce',
|
||||
'direction': 'short' if is_short else 'long',
|
||||
})
|
||||
|
||||
return closed
|
||||
|
||||
def force_reduce_all_positions(self, reduction_pct: float, current_prices: Dict[str, float]) -> List[Dict]:
|
||||
"""
|
||||
Force-reduce ALL positions by reduction_pct (e.g., close 50% of everything).
|
||||
Used when drawdown stop triggers.
|
||||
"""
|
||||
closed = []
|
||||
open_positions = self.store.get_open_positions()
|
||||
|
||||
for position in open_positions:
|
||||
symbol = position['symbol']
|
||||
price = current_prices.get(symbol)
|
||||
if price is None:
|
||||
continue
|
||||
|
||||
reduce_amount = round(abs(position['amount']) * reduction_pct, 4)
|
||||
if reduce_amount < 0.0001:
|
||||
continue
|
||||
|
||||
is_short = position.get('metadata', {}).get('direction') == 'short'
|
||||
close_side = 'buy' if is_short else 'sell'
|
||||
|
||||
try:
|
||||
self.broker.place_market_order(symbol, reduce_amount, close_side)
|
||||
logger.warning(f"FORCE REDUCE ({reduction_pct:.0%}) {reduce_amount:.2f} {symbol} @ ${price:.4f}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error force-reducing {symbol}: {e}")
|
||||
continue
|
||||
|
||||
if is_short:
|
||||
pnl = (position['entry_price'] - price) * reduce_amount
|
||||
else:
|
||||
pnl = (price - position['entry_price']) * reduce_amount
|
||||
|
||||
self.store.reduce_position(
|
||||
position['id'], reduce_amount, price, datetime.utcnow(),
|
||||
fees=reduce_amount * price * self.commission_rate
|
||||
)
|
||||
self.safety.record_trade_result(pnl * (reduce_amount / abs(position['amount'])))
|
||||
|
||||
closed.append({
|
||||
'symbol': symbol,
|
||||
'side': close_side,
|
||||
'amount': reduce_amount,
|
||||
'exit_price': price,
|
||||
'pnl': round(pnl, 2),
|
||||
'exit_reason': 'force_reduce_all',
|
||||
'direction': 'short' if is_short else 'long',
|
||||
})
|
||||
|
||||
return closed
|
||||
|
||||
def get_portfolio_state(self) -> Dict:
|
||||
@@ -385,18 +710,25 @@ class TradingExecutor:
|
||||
total_position_value = sum(p.get('market_value', 0) for p in positions)
|
||||
equity = portfolio['equity']
|
||||
|
||||
# Count directions
|
||||
num_long = sum(1 for p in open_db if p.get('metadata', {}).get('direction') == 'long')
|
||||
num_short = sum(1 for p in open_db if p.get('metadata', {}).get('direction') == 'short')
|
||||
|
||||
return {
|
||||
'equity': equity,
|
||||
'cash': portfolio['cash'],
|
||||
'position_ratio': total_position_value / equity if equity > 0 else 0,
|
||||
'unrealized_pnl': sum(p.get('unrealized_pl', 0) for p in positions) / equity
|
||||
if equity > 0 else 0,
|
||||
'time_in_position': 0, # Simplified
|
||||
'time_in_position': 0,
|
||||
'num_positions': len(positions),
|
||||
'num_long': num_long,
|
||||
'num_short': num_short,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting portfolio state: {e}")
|
||||
return {
|
||||
'equity': 0, 'cash': 0, 'position_ratio': 0,
|
||||
'unrealized_pnl': 0, 'time_in_position': 0, 'num_positions': 0,
|
||||
'num_long': 0, 'num_short': 0,
|
||||
}
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
"""
|
||||
Trade Executor
|
||||
Executes live trades via Alpaca broker with position management.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class TradingExecutor:
|
||||
"""
|
||||
Executes live trades based on RL agent decisions.
|
||||
Manages positions, stop losses, and take profits.
|
||||
"""
|
||||
|
||||
def __init__(self, broker, store, safety, config: Dict):
|
||||
"""
|
||||
Args:
|
||||
broker: AlpacaBroker instance
|
||||
store: DataStore instance
|
||||
safety: SafetyManager instance
|
||||
config: Trading configuration dict
|
||||
"""
|
||||
self.broker = broker
|
||||
self.store = store
|
||||
self.safety = safety
|
||||
self.config = config
|
||||
self.commission_rate = config.get('commission_rate', 0.001)
|
||||
|
||||
def execute_signal(self, symbol: str, action: int, current_price: float,
|
||||
strategy_params: Dict = None, combined_equity: float = None) -> Optional[Dict]:
|
||||
"""
|
||||
Execute a trading action from the RL agent.
|
||||
|
||||
Actions:
|
||||
0: Hold
|
||||
1: Buy 25% of available capital (go long)
|
||||
2: Buy 50% of available capital (go long)
|
||||
3: Close 50% of position (long or short)
|
||||
4: Close 100% of position (long or short)
|
||||
5: Short 25% of available capital
|
||||
6: Short 50% of available capital
|
||||
|
||||
Args:
|
||||
combined_equity: Optional total equity across all brokers (for multi-broker setups)
|
||||
If provided, used for safety checks instead of single-broker equity
|
||||
|
||||
Returns: trade record dict, or None if no action taken
|
||||
"""
|
||||
if action == 0: # Hold
|
||||
return None
|
||||
|
||||
if current_price <= 0:
|
||||
return None
|
||||
|
||||
strategy_params = strategy_params or {}
|
||||
|
||||
# Get portfolio state
|
||||
try:
|
||||
portfolio = self.broker.get_portfolio()
|
||||
equity = portfolio['equity']
|
||||
cash = portfolio['cash']
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting portfolio: {e}")
|
||||
return None
|
||||
|
||||
# Use combined equity for safety checks if provided (multi-broker setup)
|
||||
safety_equity = combined_equity if combined_equity is not None else equity
|
||||
|
||||
# Update safety peak
|
||||
self.safety.update_peak_equity(safety_equity)
|
||||
|
||||
# Get current positions
|
||||
open_positions = self.store.get_open_positions()
|
||||
open_for_symbol = [p for p in open_positions if p['symbol'] == symbol]
|
||||
num_positions = len(set(p['symbol'] for p in open_positions))
|
||||
|
||||
# BUY LONG actions
|
||||
if action in (1, 2):
|
||||
if open_for_symbol:
|
||||
logger.debug(f"Already have position in {symbol}, skipping buy")
|
||||
return None
|
||||
|
||||
pct = 0.25 if action == 1 else 0.50
|
||||
invest = cash * pct
|
||||
|
||||
max_position_pct = self.config.get('max_position_pct', 12) / 100
|
||||
max_invest = equity * max_position_pct
|
||||
if invest > max_invest:
|
||||
invest = max_invest
|
||||
|
||||
if invest < self.config.get('min_trade_value', 3):
|
||||
return None
|
||||
|
||||
shares = int(invest / current_price)
|
||||
if shares < 1:
|
||||
shares = round(invest / current_price, 4)
|
||||
if shares * current_price < self.config.get('min_trade_value', 3):
|
||||
return None
|
||||
|
||||
allowed, reason = self.safety.validate_trade(
|
||||
symbol, 'buy', shares, current_price, safety_equity, num_positions
|
||||
)
|
||||
if not allowed:
|
||||
logger.debug(f"Trade blocked: {reason}")
|
||||
return None
|
||||
|
||||
try:
|
||||
order = self.broker.place_market_order(symbol, shares, 'buy')
|
||||
logger.info(f"BUY {shares} {symbol} @ ~${current_price:.4f}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error placing buy order: {e}")
|
||||
return None
|
||||
|
||||
stop_loss = strategy_params.get('stop_loss')
|
||||
take_profit = strategy_params.get('take_profit')
|
||||
|
||||
trade = {
|
||||
'symbol': symbol,
|
||||
'side': 'buy',
|
||||
'amount': shares,
|
||||
'entry_price': current_price,
|
||||
'entry_time': datetime.utcnow().isoformat(),
|
||||
'strategy_id': strategy_params.get('strategy_id', 'auto'),
|
||||
'stop_loss': stop_loss,
|
||||
'take_profit': take_profit,
|
||||
'order_id': str(order.get('id', '')),
|
||||
'status': 'open',
|
||||
'metadata': {
|
||||
'action': action,
|
||||
'invest_pct': pct,
|
||||
'equity_at_entry': equity,
|
||||
'direction': 'long',
|
||||
},
|
||||
}
|
||||
trade_id = self.store.record_trade(trade)
|
||||
trade['id'] = trade_id
|
||||
return trade
|
||||
|
||||
# CLOSE POSITION actions (long or short)
|
||||
elif action in (3, 4):
|
||||
if not open_for_symbol:
|
||||
return None
|
||||
|
||||
position = open_for_symbol[0]
|
||||
total_shares = position['amount']
|
||||
is_short = position.get('metadata', {}).get('direction') == 'short'
|
||||
|
||||
if action == 3:
|
||||
close_shares = abs(total_shares) * 0.5
|
||||
else:
|
||||
close_shares = abs(total_shares)
|
||||
|
||||
close_shares = round(close_shares, 4)
|
||||
if close_shares * current_price < self.config.get('min_trade_value', 1):
|
||||
close_shares = abs(total_shares)
|
||||
|
||||
# Determine order side (opposite of position direction)
|
||||
close_side = 'buy' if is_short else 'sell'
|
||||
|
||||
try:
|
||||
order = self.broker.place_market_order(symbol, close_shares, close_side)
|
||||
logger.info(f"CLOSE({close_side.upper()}) {close_shares} {symbol} @ ~${current_price:.4f}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error placing close order: {e}")
|
||||
return None
|
||||
|
||||
# Calculate P&L
|
||||
if is_short:
|
||||
pnl = (position['entry_price'] - current_price) * close_shares
|
||||
else:
|
||||
pnl = (current_price - position['entry_price']) * close_shares
|
||||
|
||||
if close_shares >= abs(total_shares) * 0.99:
|
||||
self.store.close_position(
|
||||
position['id'], current_price, datetime.utcnow(),
|
||||
fees=close_shares * current_price * self.commission_rate
|
||||
)
|
||||
self.safety.record_trade_result(pnl)
|
||||
|
||||
return {
|
||||
'symbol': symbol,
|
||||
'side': close_side,
|
||||
'amount': close_shares,
|
||||
'exit_price': current_price,
|
||||
'pnl': round(pnl, 2),
|
||||
'pnl_pct': round(pnl / (position['entry_price'] * close_shares) * 100, 2),
|
||||
'action': action,
|
||||
'direction': 'short' if is_short else 'long',
|
||||
}
|
||||
else:
|
||||
self.safety.record_trade_result(pnl)
|
||||
self.store.close_position(
|
||||
position['id'], current_price, datetime.utcnow(),
|
||||
fees=close_shares * current_price * self.commission_rate
|
||||
)
|
||||
|
||||
remaining = abs(total_shares) - close_shares
|
||||
if remaining > 0:
|
||||
self.store.record_trade({
|
||||
'symbol': symbol,
|
||||
'side': position['side'],
|
||||
'amount': remaining,
|
||||
'entry_price': position['entry_price'],
|
||||
'entry_time': position['entry_time'],
|
||||
'strategy_id': position.get('strategy_id', 'auto'),
|
||||
'stop_loss': position.get('stop_loss'),
|
||||
'take_profit': position.get('take_profit'),
|
||||
'status': 'open',
|
||||
'metadata': position.get('metadata', {}),
|
||||
})
|
||||
|
||||
return {
|
||||
'symbol': symbol,
|
||||
'side': close_side,
|
||||
'amount': close_shares,
|
||||
'exit_price': current_price,
|
||||
'pnl': round(pnl, 2),
|
||||
'action': action,
|
||||
'direction': 'short' if is_short else 'long',
|
||||
}
|
||||
|
||||
# SHORT actions
|
||||
elif action in (5, 6):
|
||||
if open_for_symbol:
|
||||
logger.debug(f"Already have position in {symbol}, skipping short")
|
||||
return None
|
||||
|
||||
pct = 0.25 if action == 5 else 0.50
|
||||
invest = cash * pct
|
||||
|
||||
max_position_pct = self.config.get('max_position_pct', 12) / 100
|
||||
max_invest = equity * max_position_pct
|
||||
if invest > max_invest:
|
||||
invest = max_invest
|
||||
|
||||
if invest < self.config.get('min_trade_value', 3):
|
||||
return None
|
||||
|
||||
shares = int(invest / current_price)
|
||||
if shares < 1:
|
||||
shares = round(invest / current_price, 4)
|
||||
if shares * current_price < self.config.get('min_trade_value', 3):
|
||||
return None
|
||||
|
||||
allowed, reason = self.safety.validate_trade(
|
||||
symbol, 'sell', shares, current_price, safety_equity, num_positions
|
||||
)
|
||||
if not allowed:
|
||||
logger.debug(f"Short blocked: {reason}")
|
||||
return None
|
||||
|
||||
try:
|
||||
order = self.broker.place_market_order(symbol, shares, 'sell')
|
||||
logger.info(f"SHORT {shares} {symbol} @ ~${current_price:.4f}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error placing short order: {e}")
|
||||
return None
|
||||
|
||||
# For shorts, stop_loss is ABOVE entry, take_profit is BELOW
|
||||
stop_loss = strategy_params.get('short_stop_loss') or strategy_params.get('stop_loss')
|
||||
take_profit = strategy_params.get('short_take_profit') or strategy_params.get('take_profit')
|
||||
|
||||
trade = {
|
||||
'symbol': symbol,
|
||||
'side': 'sell',
|
||||
'amount': shares,
|
||||
'entry_price': current_price,
|
||||
'entry_time': datetime.utcnow().isoformat(),
|
||||
'strategy_id': strategy_params.get('strategy_id', 'auto'),
|
||||
'stop_loss': stop_loss,
|
||||
'take_profit': take_profit,
|
||||
'order_id': str(order.get('id', '')),
|
||||
'status': 'open',
|
||||
'metadata': {
|
||||
'action': action,
|
||||
'invest_pct': pct,
|
||||
'equity_at_entry': equity,
|
||||
'direction': 'short',
|
||||
},
|
||||
}
|
||||
trade_id = self.store.record_trade(trade)
|
||||
trade['id'] = trade_id
|
||||
return trade
|
||||
|
||||
return None
|
||||
|
||||
def check_exits(self, current_prices: Dict[str, float]) -> List[Dict]:
|
||||
"""
|
||||
Check all open positions for stop loss / take profit exits.
|
||||
Handles both long and short positions.
|
||||
Returns list of closed trades.
|
||||
"""
|
||||
closed = []
|
||||
open_positions = self.store.get_open_positions()
|
||||
|
||||
for position in open_positions:
|
||||
symbol = position['symbol']
|
||||
price = current_prices.get(symbol)
|
||||
if price is None:
|
||||
continue
|
||||
|
||||
should_exit = False
|
||||
exit_reason = ''
|
||||
is_short = position.get('metadata', {}).get('direction') == 'short'
|
||||
|
||||
if is_short:
|
||||
# Short position: stop_loss is ABOVE entry, take_profit is BELOW
|
||||
if position.get('stop_loss') and price >= position['stop_loss']:
|
||||
should_exit = True
|
||||
exit_reason = 'stop_loss'
|
||||
elif position.get('take_profit') and price <= position['take_profit']:
|
||||
should_exit = True
|
||||
exit_reason = 'take_profit'
|
||||
else:
|
||||
# Long position: stop_loss is BELOW entry, take_profit is ABOVE
|
||||
if position.get('stop_loss') and price <= position['stop_loss']:
|
||||
should_exit = True
|
||||
exit_reason = 'stop_loss'
|
||||
elif position.get('take_profit') and price >= position['take_profit']:
|
||||
should_exit = True
|
||||
exit_reason = 'take_profit'
|
||||
|
||||
if should_exit:
|
||||
# Close side is opposite of position direction
|
||||
close_side = 'buy' if is_short else 'sell'
|
||||
try:
|
||||
self.broker.place_market_order(
|
||||
symbol, position['amount'], close_side
|
||||
)
|
||||
logger.info(f"EXIT ({exit_reason}) {symbol} @ ${price:.4f} [{'SHORT' if is_short else 'LONG'}]")
|
||||
except Exception as e:
|
||||
logger.error(f"Error executing exit for {symbol}: {e}")
|
||||
continue
|
||||
|
||||
if is_short:
|
||||
pnl = (position['entry_price'] - price) * position['amount']
|
||||
else:
|
||||
pnl = (price - position['entry_price']) * position['amount']
|
||||
|
||||
self.store.close_position(
|
||||
position['id'], price, datetime.utcnow(),
|
||||
fees=position['amount'] * price * self.commission_rate
|
||||
)
|
||||
self.safety.record_trade_result(pnl)
|
||||
|
||||
closed.append({
|
||||
'symbol': symbol,
|
||||
'side': close_side,
|
||||
'amount': position['amount'],
|
||||
'entry_price': position['entry_price'],
|
||||
'exit_price': price,
|
||||
'pnl': round(pnl, 2),
|
||||
'pnl_pct': round(pnl / (position['entry_price'] * position['amount']) * 100, 2),
|
||||
'exit_reason': exit_reason,
|
||||
'direction': 'short' if is_short else 'long',
|
||||
})
|
||||
|
||||
return closed
|
||||
|
||||
def get_portfolio_state(self) -> Dict:
|
||||
"""Get current portfolio state for RL agent"""
|
||||
try:
|
||||
portfolio = self.broker.get_portfolio()
|
||||
positions = self.broker.get_positions()
|
||||
open_db = self.store.get_open_positions()
|
||||
|
||||
total_position_value = sum(p.get('market_value', 0) for p in positions)
|
||||
equity = portfolio['equity']
|
||||
|
||||
return {
|
||||
'equity': equity,
|
||||
'cash': portfolio['cash'],
|
||||
'position_ratio': total_position_value / equity if equity > 0 else 0,
|
||||
'unrealized_pnl': sum(p.get('unrealized_pl', 0) for p in positions) / equity
|
||||
if equity > 0 else 0,
|
||||
'time_in_position': 0, # Simplified
|
||||
'num_positions': len(positions),
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting portfolio state: {e}")
|
||||
return {
|
||||
'equity': 0, 'cash': 0, 'position_ratio': 0,
|
||||
'unrealized_pnl': 0, 'time_in_position': 0, 'num_positions': 0,
|
||||
}
|
||||
Reference in New Issue
Block a user