Skip to content

Commit 077948a

Browse files
fix: stop cross-attention lag from wrapping short windows
When the lag exceeded the window length, the history slice used a negative stop and either crashed or read the end of the same window. Lags that reach past the start of the window now see an empty history. Co-authored-by: quantum-like <[email protected]>
1 parent 7312029 commit 077948a

2 files changed

Lines changed: 37 additions & 13 deletions

File tree

‎src/ml4t/models/portfolio/components.py‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -179,18 +179,24 @@ def __init__(self, *, d_model: int, n_heads: int, dropout: float, lag: int) -> N
179179
def forward(self, x: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
180180
batch_size, n_periods, n_assets, d_model = x.shape
181181
if self.lag > 0:
182-
pad = torch.zeros(
183-
(batch_size, self.lag, n_assets, d_model),
184-
device=x.device,
185-
dtype=x.dtype,
186-
)
187-
kv = torch.cat([pad, x[:, : n_periods - self.lag]], dim=1)
188-
pad_mask = torch.zeros(
189-
(batch_size, self.lag, n_assets),
190-
device=mask.device,
191-
dtype=mask.dtype,
192-
)
193-
kv_mask = torch.cat([pad_mask, mask[:, : n_periods - self.lag]], dim=1)
182+
history = n_periods - self.lag
183+
if history <= 0:
184+
# A negative slice would wrap to the end of the window.
185+
kv = torch.zeros_like(x)
186+
kv_mask = torch.zeros_like(mask)
187+
else:
188+
pad = torch.zeros(
189+
(batch_size, self.lag, n_assets, d_model),
190+
device=x.device,
191+
dtype=x.dtype,
192+
)
193+
kv = torch.cat([pad, x[:, :history]], dim=1)
194+
pad_mask = torch.zeros(
195+
(batch_size, self.lag, n_assets),
196+
device=mask.device,
197+
dtype=mask.dtype,
198+
)
199+
kv_mask = torch.cat([pad_mask, mask[:, :history]], dim=1)
194200
else:
195201
kv = x
196202
kv_mask = mask

‎tests/test_financial_correctness.py‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from ml4t.models.latent_factors.ipca import _normalize_theta_y
1717
from ml4t.models.latent_factors.rp_pca import _risk_premium_matrix
1818
from ml4t.models.portfolio import runtime as portfolio_runtime
19-
from ml4t.models.portfolio.components import MacroGraphAttention
19+
from ml4t.models.portfolio.components import CrossSectionalAttention, MacroGraphAttention
2020
from ml4t.models.portfolio.losses import compute_net_portfolio_returns, softmin_sharpe
2121
from ml4t.models.portfolio.postprocessors import (
2222
apply_turnover_limit,
@@ -330,6 +330,24 @@ def fake_validation(
330330
)
331331

332332

333+
def test_cross_attention_lag_longer_than_the_window_does_not_wrap() -> None:
334+
torch.manual_seed(0)
335+
window = torch.randn(1, 3, 2, 4)
336+
mask = torch.ones(1, 3, 2)
337+
matched = CrossSectionalAttention(d_model=4, n_heads=1, dropout=0.0, lag=3)
338+
longer = CrossSectionalAttention(d_model=4, n_heads=1, dropout=0.0, lag=8)
339+
longer.load_state_dict(matched.state_dict())
340+
matched.eval()
341+
longer.eval()
342+
343+
matched_output = matched(window, mask)
344+
longer_output = longer(window, mask)
345+
346+
assert matched_output.shape == window.shape
347+
assert torch.isfinite(longer_output).all()
348+
torch.testing.assert_close(longer_output, matched_output)
349+
350+
333351
def test_graph_attention_stays_finite_when_a_node_has_no_neighbors() -> None:
334352
torch.manual_seed(0)
335353
layer = MacroGraphAttention(

0 commit comments

Comments
 (0)