fix(llama.cpp): set -1 as default for max tokens (#2087)

Signed-off-by: Ettore Di Giacinto <[email protected]>
mudler · Apr 20, 2024 · afa1bca · afa1bca
1 parent 03adc1f
commit afa1bca
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/core/config/backend_config.go b/core/config/backend_config.go
@@ -205,15 +205,15 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
 	defaultTopP := 0.95
 	defaultTopK := 40
 	defaultTemp := 0.9
-	defaultMaxTokens := 2048
 	defaultMirostat := 2
 	defaultMirostatTAU := 5.0
 	defaultMirostatETA := 0.1
 	defaultTypicalP := 1.0
 	defaultTFZ := 1.0
+	defaultInfinity := -1
 
 	// Try to offload all GPU layers (if GPU is found)
-	defaultNGPULayers := 99999999
+	defaultHigh := 99999999
 
 	trueV := true
 	falseV := false
@@ -254,7 +254,7 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
 	}
 
 	if cfg.Maxtokens == nil {
-		cfg.Maxtokens = &defaultMaxTokens
+		cfg.Maxtokens = &defaultInfinity
 	}
 
 	if cfg.Mirostat == nil {
@@ -269,7 +269,7 @@ func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
 		cfg.MirostatTAU = &defaultMirostatTAU
 	}
 	if cfg.NGPULayers == nil {
-		cfg.NGPULayers = &defaultNGPULayers
+		cfg.NGPULayers = &defaultHigh
 	}
 
 	if cfg.LowVRAM == nil {