Autodetect rms_norm_eps and n_gqa for llama-2-70b

oobabooga · Jul 24, 2023 · 08c622d · 08c622d
1 parent a07d070
commit 08c622d
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/models/config.yaml b/models/config.yaml
@@ -277,3 +277,6 @@ TheBloke_WizardLM-30B-GPTQ:
 .*llama-(2|v2).*chat:
  mode: 'instruct'
  instruction_template: 'Llama-v2'
+.*llama.*70b.*ggml.*\.bin:
+ n_gqa: 8
+ rms_norm_eps: 1.0e-5
diff --git a/server.py b/server.py
@@ -1104,6 +1104,8 @@ def create_interface():
  'skip_special_tokens': shared.settings['skip_special_tokens'],
  'custom_stopping_strings': shared.settings['custom_stopping_strings'],
  'truncation_length': shared.settings['truncation_length'],
+ 'n_gqa': 0,
+ 'rms_norm_eps': 0,
  }
 
  shared.model_config.move_to_end('.*', last=False) # Move to the beginning