fix max seq len (#807)

pytorch · Apr 19, 2024 · 41341fd · 41341fd
1 parent b74fd3a
commit 41341fd
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/torchtune/models/llama3/_model_builders.py b/torchtune/models/llama3/_model_builders.py
@@ -36,7 +36,7 @@ def llama3_8b() -> TransformerDecoder:
  num_heads=32,
  num_kv_heads=8,
  embed_dim=4096,
- max_seq_len=4096,
+ max_seq_len=8192,
  intermediate_dim=14336,
  attn_dropout=0.0,
  norm_eps=1e-5,
@@ -89,7 +89,7 @@ def lora_llama3_8b(
  num_heads=32,
  num_kv_heads=8,
  embed_dim=4096,
- max_seq_len=4096,
+ max_seq_len=8192,
  intermediate_dim=14336,
  attn_dropout=0.0,
  norm_eps=1e-5,