Skip to content

Commit bbb40a0

Browse files
authored
server: context shift for context windows larger than 8k, add error when hitting context limit (#16712)
1 parent 993acc7 commit bbb40a0

4 files changed

Lines changed: 34 additions & 62 deletions

File tree

‎llm/llama_server.go‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -273,28 +273,15 @@ func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req
273273
return nil, err
274274
}
275275

276-
// llama-server rejects prompts that fill the entire slot context, while the
277-
// old runner could accept exactly num_ctx prompt tokens. Keep one token of
278-
// headroom so token-level truncation preserves old behavior as closely as
279-
// llama-server allows.
280-
limit := s.options.NumCtx - 1
281-
if len(tokens) <= limit {
282-
return prompt, nil
283-
}
284-
285-
nKeep := req.Options.NumKeep
286-
if nKeep < 0 {
287-
nKeep = len(tokens)
276+
// s.options.NumCtx is the runner's effective per-slot context length. It
277+
// has already been capped by the model's training context during launch.
278+
if len(tokens) >= s.options.NumCtx {
279+
return nil, api.StatusError{
280+
StatusCode: http.StatusBadRequest,
281+
ErrorMessage: "the prompt is longer than the context length currently available to the model; shorten the prompt or adjust the context length in settings",
282+
}
288283
}
289-
nKeep = min(nKeep, limit)
290-
291-
discard := len(tokens) - limit
292-
truncated := make([]int, 0, limit)
293-
truncated = append(truncated, tokens[:nKeep]...)
294-
truncated = append(truncated, tokens[nKeep+discard:]...)
295-
296-
slog.Warn("truncating input prompt", "limit", s.options.NumCtx, "prompt", len(tokens), "keep", nKeep, "new", len(truncated))
297-
return truncated, nil
284+
return prompt, nil
298285
}
299286

300287
func (s *llamaServerRunner) ContextLength() int {

‎llm/llama_server_test.go‎

Lines changed: 16 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -488,14 +488,16 @@ func TestLlamaServerCompletionForwardsRepeatLastNZero(t *testing.T) {
488488
}
489489
}
490490

491-
func TestLlamaServerCompletionTruncatesPromptAsTokens(t *testing.T) {
492-
var completionReq llamaServerCompletionRequest
491+
func TestLlamaServerCompletionRejectsPromptOverContext(t *testing.T) {
492+
const wantError = "the prompt is longer than the context length currently available to the model; shorten the prompt or adjust the context length in settings"
493+
493494
var tokenizeReq struct {
494495
Content string `json:"content"`
495496
AddSpecial bool `json:"add_special"`
496497
ParseSpecial *bool `json:"parse_special"`
497498
}
498499

500+
completionCalled := false
499501
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
500502
switch r.URL.Path {
501503
case "/health":
@@ -507,10 +509,7 @@ func TestLlamaServerCompletionTruncatesPromptAsTokens(t *testing.T) {
507509
}
508510
fmt.Fprint(w, `{"tokens":[0,1,2,3,4,5,6,7,8,9]}`)
509511
case "/completion":
510-
if err := json.NewDecoder(r.Body).Decode(&completionReq); err != nil {
511-
t.Errorf("invalid completion request body: %v", err)
512-
return
513-
}
512+
completionCalled = true
514513
w.Header().Set("Content-Type", "text/event-stream")
515514
fmt.Fprintln(w, `data: {"content":"ok","stop":true,"timings":{"prompt_n":7,"prompt_ms":1,"predicted_n":1,"predicted_ms":1}}`)
516515
default:
@@ -537,8 +536,15 @@ func TestLlamaServerCompletionTruncatesPromptAsTokens(t *testing.T) {
537536
Options: &opts,
538537
Truncate: true,
539538
}, func(cr CompletionResponse) {})
540-
if err != nil {
541-
t.Fatalf("Completion error: %v", err)
539+
var statusErr api.StatusError
540+
if !errors.As(err, &statusErr) {
541+
t.Fatalf("Completion error = %T %v, want api.StatusError", err, err)
542+
}
543+
if statusErr.StatusCode != http.StatusBadRequest {
544+
t.Fatalf("StatusCode = %d, want %d", statusErr.StatusCode, http.StatusBadRequest)
545+
}
546+
if statusErr.ErrorMessage != wantError {
547+
t.Fatalf("ErrorMessage = %q, want %q", statusErr.ErrorMessage, wantError)
542548
}
543549

544550
if tokenizeReq.Content != strings.Repeat("long prompt ", 2) {
@@ -547,20 +553,8 @@ func TestLlamaServerCompletionTruncatesPromptAsTokens(t *testing.T) {
547553
if !tokenizeReq.AddSpecial {
548554
t.Fatal("expected tokenize request to add special tokens")
549555
}
550-
551-
got, ok := completionReq.Prompt.([]any)
552-
if !ok {
553-
t.Fatalf("completion prompt = %T, want token array", completionReq.Prompt)
554-
}
555-
want := []int{0, 1, 2, 6, 7, 8, 9}
556-
if len(got) != len(want) {
557-
t.Fatalf("token prompt len = %d, want %d: %#v", len(got), len(want), got)
558-
}
559-
for i, wantToken := range want {
560-
gotToken, ok := got[i].(float64)
561-
if !ok || int(gotToken) != wantToken {
562-
t.Fatalf("token prompt[%d] = %#v, want %d", i, got[i], wantToken)
563-
}
556+
if completionCalled {
557+
t.Fatal("completion endpoint was called")
564558
}
565559
}
566560

‎server/sched.go‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -132,14 +132,8 @@ func (s *Scheduler) GetRunner(c context.Context, m *Model, opts api.Options, ses
132132
return s.getRunner(c, m, opts, sessionDuration, false, false, nil)
133133
}
134134

135-
const contextShiftSmallContextLimit = 8192
136-
137-
func resolveContextShift(shift *bool, numCtx int) bool {
138-
if shift != nil {
139-
return *shift
140-
}
141-
142-
return numCtx > 0 && numCtx < contextShiftSmallContextLimit
135+
func resolveContextShift(shift *bool) bool {
136+
return shift == nil || *shift
143137
}
144138

145139
func effectiveModelContext(numCtx int, f *ggml.GGML) int {
@@ -174,7 +168,7 @@ func (s *Scheduler) getRunner(c context.Context, m *Model, opts api.Options, ses
174168

175169
contextShift := false
176170
if m.ModelPath != "" {
177-
contextShift = resolveContextShift(shift, opts.NumCtx)
171+
contextShift = resolveContextShift(shift)
178172
}
179173

180174
req := &LlmRequest{
@@ -566,7 +560,7 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.De
566560
}
567561

568562
launchOpts = s.applyLlamaServerMmapDefaults(req, launchOpts, systemInfo, loadGpus, f, numParallel)
569-
req.contextShift = resolveContextShift(req.shift, effectiveModelContext(launchOpts.NumCtx, f))
563+
req.contextShift = resolveContextShift(req.shift)
570564

571565
config := llamaServerConfigForModel(req.model)
572566
config.ContextShift = req.contextShift
@@ -691,7 +685,7 @@ iGPUScan:
691685
trainContext := modelTrainContext(f)
692686
if effectiveNumCtx := llama.ContextLength(); req.model.ModelPath != "" && effectiveNumCtx > 0 {
693687
req.opts.NumCtx = effectiveNumCtx
694-
req.contextShift = resolveContextShift(req.shift, effectiveNumCtx)
688+
req.contextShift = resolveContextShift(req.shift)
695689
}
696690
runner := &runnerRef{
697691
model: req.model,
@@ -1420,7 +1414,7 @@ func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) bool
14201414

14211415
contextShift := req.contextShift
14221416
if req.model.ModelPath != "" {
1423-
contextShift = resolveContextShift(req.shift, optsNew.NumCtx)
1417+
contextShift = resolveContextShift(req.shift)
14241418
}
14251419
if runner.contextShift != contextShift {
14261420
return true

‎server/sched_test.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -907,19 +907,16 @@ func TestResolveContextShift(t *testing.T) {
907907
tests := []struct {
908908
name string
909909
shift *bool
910-
ctx int
911910
want bool
912911
}{
913-
{name: "unset small context keeps legacy shift", ctx: 128, want: true},
914-
{name: "unset large context disables shift", ctx: contextShiftSmallContextLimit, want: false},
915-
{name: "unset invalid context disables shift", ctx: 0, want: false},
916-
{name: "explicit false wins for small context", shift: &falseValue, ctx: 128, want: false},
917-
{name: "explicit true wins for large context", shift: &trueValue, ctx: 32768, want: true},
912+
{name: "unset defaults to shift", want: true},
913+
{name: "explicit false disables shift", shift: &falseValue, want: false},
914+
{name: "explicit true enables shift", shift: &trueValue, want: true},
918915
}
919916

920917
for _, tt := range tests {
921918
t.Run(tt.name, func(t *testing.T) {
922-
require.Equal(t, tt.want, resolveContextShift(tt.shift, tt.ctx))
919+
require.Equal(t, tt.want, resolveContextShift(tt.shift))
923920
})
924921
}
925922
}

0 commit comments

Comments
 (0)