Ver código fonte

dflash A/B: attention_backend B12X→FLASH_ATTN (B12X rejects the draft's block_size; trying the standard backend)

Alexander Korolev 5 dias atrás
pai
commit
a3004e8e0b
1 arquivos alterados com 1 adições e 1 exclusões
  1. 1 1
      recipes/glm53-flash-fp4local-2x-dflash/recipe.toml

+ 1 - 1
recipes/glm53-flash-fp4local-2x-dflash/recipe.toml

@@ -63,7 +63,7 @@ args = [
     "--enable-auto-tool-choice",
     "--gpu-memory-utilization", "0.87",
     "--speculative-config", """
-'{"method":"dflash","model":"{{dflash2}}","num_speculative_tokens":7,"rejection_sample_method":"standard","attention_backend":"B12X","draft_load_config":{"load_format":"auto"}}'""",
+'{"method":"dflash","model":"{{dflash2}}","num_speculative_tokens":7,"rejection_sample_method":"standard","attention_backend":"FLASH_ATTN","draft_load_config":{"load_format":"auto"}}'""",
 ]
 
 # DFlash2 drafter (A/B vs the card's embedded MTP head): distilled against the