Skip to content

Commit 0f047fe

Browse files
authored
llm: context shift allow shiftable prompts (#16764)
1 parent 9e4ed74 commit 0f047fe

5 files changed

Lines changed: 209 additions & 16 deletions

File tree

‎integration/context_test.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@ import (
1717
func TestLongInputContext(t *testing.T) {
1818
// Setting NUM_PARALLEL to 1 ensures the allocated context is exactly what
1919
// we asked for and there is nothing extra that we could spill over into.
20-
// Older runners silently truncate oversized prompts, while llama-server
21-
// rejects them with a client error. Accept both behaviors so this test can
22-
// run against main and the llama-server branch.
20+
// Context shift happens after a prompt has been admitted to a slot. Initial
21+
// prompts that fill or exceed the slot are still rejected by llama-server.
22+
// Accept a context-limit error here because older runners may truncate this
23+
// prompt while llama-server reports it as too large to admit.
2324
t.Setenv("OLLAMA_NUM_PARALLEL", "1")
2425

2526
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
@@ -74,6 +75,7 @@ func isContextLimitError(err string) bool {
7475
return strings.Contains(err, "context") &&
7576
(strings.Contains(err, "exceed") ||
7677
strings.Contains(err, "too large") ||
78+
strings.Contains(err, "longer") ||
7779
strings.Contains(err, "too long"))
7880
}
7981

‎llm/llama_server.go‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -273,15 +273,31 @@ func (s *llamaServerRunner) completionPromptForRequest(ctx context.Context, req
273273
return nil, err
274274
}
275275

276-
// s.options.NumCtx is the runner's effective per-slot context length. It
277-
// has already been capped by the model's training context during launch.
278-
if len(tokens) >= s.options.NumCtx {
276+
limit := s.options.NumCtx - 1
277+
if len(tokens) <= limit {
278+
return prompt, nil
279+
}
280+
281+
if !s.launch.config.ContextShift {
279282
return nil, api.StatusError{
280283
StatusCode: http.StatusBadRequest,
281-
ErrorMessage: "the prompt is longer than the context length currently available to the model; shorten the prompt or adjust the context length in settings",
284+
ErrorMessage: "the prompt is longer than the context length currently available to the model; shorten the prompt, adjust the context length in settings, or use a model with a longer context length",
282285
}
283286
}
284-
return prompt, nil
287+
288+
nKeep := req.Options.NumKeep
289+
if nKeep < 0 {
290+
nKeep = len(tokens)
291+
}
292+
nKeep = min(nKeep, limit)
293+
294+
discard := len(tokens) - limit
295+
truncated := make([]int, 0, limit)
296+
truncated = append(truncated, tokens[:nKeep]...)
297+
truncated = append(truncated, tokens[nKeep+discard:]...)
298+
299+
slog.Warn("truncating input prompt", "limit", limit, "prompt", len(tokens), "keep", nKeep, "new", len(truncated))
300+
return truncated, nil
285301
}
286302

287303
func (s *llamaServerRunner) ContextLength() int {

‎llm/llama_server_test.go‎

Lines changed: 155 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,7 @@ func TestLlamaServerCompletionForwardsRepeatLastNZero(t *testing.T) {
489489
}
490490

491491
func TestLlamaServerCompletionRejectsPromptOverContext(t *testing.T) {
492-
const wantError = "the prompt is longer than the context length currently available to the model; shorten the prompt or adjust the context length in settings"
492+
const wantError = "the prompt is longer than the context length currently available to the model; shorten the prompt, adjust the context length in settings, or use a model with a longer context length"
493493

494494
var tokenizeReq struct {
495495
Content string `json:"content"`
@@ -558,6 +558,160 @@ func TestLlamaServerCompletionRejectsPromptOverContext(t *testing.T) {
558558
}
559559
}
560560

561+
func TestLlamaServerCompletionContextShiftAllowsPromptWithHeadroom(t *testing.T) {
562+
var capturedReq llamaServerCompletionRequest
563+
var tokenizeReq struct {
564+
Content string `json:"content"`
565+
AddSpecial bool `json:"add_special"`
566+
ParseSpecial *bool `json:"parse_special"`
567+
}
568+
569+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
570+
switch r.URL.Path {
571+
case "/health":
572+
fmt.Fprint(w, `{"status":"ok"}`)
573+
case "/tokenize":
574+
if err := json.NewDecoder(r.Body).Decode(&tokenizeReq); err != nil {
575+
t.Errorf("invalid tokenize request body: %v", err)
576+
return
577+
}
578+
fmt.Fprint(w, `{"tokens":[0,1,2,3,4,5,6]}`)
579+
case "/completion":
580+
if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil {
581+
t.Errorf("invalid completion request body: %v", err)
582+
return
583+
}
584+
w.Header().Set("Content-Type", "text/event-stream")
585+
fmt.Fprintln(w, `data: {"content":"ok","stop":true,"timings":{"prompt_n":10,"prompt_ms":1,"predicted_n":1,"predicted_ms":1}}`)
586+
default:
587+
t.Errorf("unexpected path: %s", r.URL.Path)
588+
}
589+
}))
590+
defer srv.Close()
591+
592+
parts := strings.Split(srv.URL, ":")
593+
var portInt int
594+
fmt.Sscanf(parts[len(parts)-1], "%d", &portInt)
595+
596+
runner := &llamaServerRunner{
597+
port: portInt,
598+
cmd: fakeRunningCmd(),
599+
sem: semaphore.NewWeighted(1),
600+
options: api.Options{Runner: api.Runner{NumCtx: 8}},
601+
launch: llamaServerLaunchConfig{
602+
config: LlamaServerConfig{ContextShift: true},
603+
},
604+
}
605+
606+
opts := api.DefaultOptions()
607+
opts.NumKeep = 3
608+
prompt := strings.Repeat("long prompt ", 2)
609+
err := runner.Completion(t.Context(), CompletionRequest{
610+
Prompt: prompt,
611+
Options: &opts,
612+
Truncate: true,
613+
}, func(cr CompletionResponse) {})
614+
if err != nil {
615+
t.Fatalf("Completion error: %v", err)
616+
}
617+
618+
if tokenizeReq.Content != prompt {
619+
t.Fatalf("tokenize content = %q, want %q", tokenizeReq.Content, prompt)
620+
}
621+
if !tokenizeReq.AddSpecial {
622+
t.Fatal("expected tokenize request to add special tokens")
623+
}
624+
if capturedReq.Prompt != prompt {
625+
t.Fatalf("prompt = %q, want %q", capturedReq.Prompt, prompt)
626+
}
627+
if capturedReq.NKeep != opts.NumKeep {
628+
t.Fatalf("n_keep = %d, want %d", capturedReq.NKeep, opts.NumKeep)
629+
}
630+
}
631+
632+
func TestLlamaServerCompletionContextShiftTruncatesPromptOverContext(t *testing.T) {
633+
var capturedReq llamaServerCompletionRequest
634+
var tokenizeReq struct {
635+
Content string `json:"content"`
636+
AddSpecial bool `json:"add_special"`
637+
ParseSpecial *bool `json:"parse_special"`
638+
}
639+
640+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
641+
switch r.URL.Path {
642+
case "/health":
643+
fmt.Fprint(w, `{"status":"ok"}`)
644+
case "/tokenize":
645+
if err := json.NewDecoder(r.Body).Decode(&tokenizeReq); err != nil {
646+
t.Errorf("invalid tokenize request body: %v", err)
647+
return
648+
}
649+
fmt.Fprint(w, `{"tokens":[0,1,2,3,4,5,6,7,8,9]}`)
650+
case "/completion":
651+
if err := json.NewDecoder(r.Body).Decode(&capturedReq); err != nil {
652+
t.Errorf("invalid completion request body: %v", err)
653+
return
654+
}
655+
w.Header().Set("Content-Type", "text/event-stream")
656+
fmt.Fprintln(w, `data: {"content":"ok","stop":true,"timings":{"prompt_n":7,"prompt_ms":1,"predicted_n":1,"predicted_ms":1}}`)
657+
default:
658+
t.Errorf("unexpected path: %s", r.URL.Path)
659+
}
660+
}))
661+
defer srv.Close()
662+
663+
parts := strings.Split(srv.URL, ":")
664+
var portInt int
665+
fmt.Sscanf(parts[len(parts)-1], "%d", &portInt)
666+
667+
runner := &llamaServerRunner{
668+
port: portInt,
669+
cmd: fakeRunningCmd(),
670+
sem: semaphore.NewWeighted(1),
671+
options: api.Options{Runner: api.Runner{NumCtx: 8}},
672+
launch: llamaServerLaunchConfig{
673+
config: LlamaServerConfig{ContextShift: true},
674+
},
675+
}
676+
677+
opts := api.DefaultOptions()
678+
opts.NumKeep = 3
679+
prompt := strings.Repeat("long prompt ", 2)
680+
err := runner.Completion(t.Context(), CompletionRequest{
681+
Prompt: prompt,
682+
Options: &opts,
683+
Truncate: true,
684+
}, func(cr CompletionResponse) {})
685+
if err != nil {
686+
t.Fatalf("Completion error: %v", err)
687+
}
688+
689+
if tokenizeReq.Content != prompt {
690+
t.Fatalf("tokenize content = %q, want %q", tokenizeReq.Content, prompt)
691+
}
692+
if !tokenizeReq.AddSpecial {
693+
t.Fatal("expected tokenize request to add special tokens")
694+
}
695+
696+
got, ok := capturedReq.Prompt.([]any)
697+
if !ok {
698+
t.Fatalf("completion prompt = %T, want token array", capturedReq.Prompt)
699+
}
700+
want := []int{0, 1, 2, 6, 7, 8, 9}
701+
if len(got) != len(want) {
702+
t.Fatalf("token prompt len = %d, want %d: %#v", len(got), len(want), got)
703+
}
704+
for i, wantToken := range want {
705+
gotToken, ok := got[i].(float64)
706+
if !ok || int(gotToken) != wantToken {
707+
t.Fatalf("token prompt[%d] = %#v, want %d", i, got[i], wantToken)
708+
}
709+
}
710+
if capturedReq.NKeep != opts.NumKeep {
711+
t.Fatalf("n_keep = %d, want %d", capturedReq.NKeep, opts.NumKeep)
712+
}
713+
}
714+
561715
func TestLlamaServerCompletionWithMediaUsesRunnerMarker(t *testing.T) {
562716
var capturedReq llamaServerCompletionRequest
563717

‎server/sched.go‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,24 @@ func (s *Scheduler) GetRunner(c context.Context, m *Model, opts api.Options, ses
132132
return s.getRunner(c, m, opts, sessionDuration, false, false, nil)
133133
}
134134

135-
func resolveContextShift(shift *bool) bool {
136-
return shift == nil || *shift
135+
func resolveContextShift(shift *bool, m *Model) bool {
136+
if shift != nil {
137+
return *shift
138+
}
139+
140+
return supportsContextShift(m)
141+
}
142+
143+
func supportsContextShift(m *Model) bool {
144+
if m == nil {
145+
return true
146+
}
147+
148+
if m.Config.ModelFamily == "deepseek2" || slices.Contains(m.Config.ModelFamilies, "deepseek2") {
149+
return false
150+
}
151+
152+
return true
137153
}
138154

139155
func effectiveModelContext(numCtx int, f *ggml.GGML) int {
@@ -168,7 +184,7 @@ func (s *Scheduler) getRunner(c context.Context, m *Model, opts api.Options, ses
168184

169185
contextShift := false
170186
if m.ModelPath != "" {
171-
contextShift = resolveContextShift(shift)
187+
contextShift = resolveContextShift(shift, m)
172188
}
173189

174190
req := &LlmRequest{
@@ -560,7 +576,7 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.De
560576
}
561577

562578
launchOpts = s.applyLlamaServerMmapDefaults(req, launchOpts, systemInfo, loadGpus, f, numParallel)
563-
req.contextShift = resolveContextShift(req.shift)
579+
req.contextShift = resolveContextShift(req.shift, req.model)
564580

565581
config := llamaServerConfigForModel(req.model)
566582
config.ContextShift = req.contextShift
@@ -685,7 +701,7 @@ iGPUScan:
685701
trainContext := modelTrainContext(f)
686702
if effectiveNumCtx := llama.ContextLength(); req.model.ModelPath != "" && effectiveNumCtx > 0 {
687703
req.opts.NumCtx = effectiveNumCtx
688-
req.contextShift = resolveContextShift(req.shift)
704+
req.contextShift = resolveContextShift(req.shift, req.model)
689705
}
690706
runner := &runnerRef{
691707
model: req.model,
@@ -1414,7 +1430,7 @@ func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) bool
14141430

14151431
contextShift := req.contextShift
14161432
if req.model.ModelPath != "" {
1417-
contextShift = resolveContextShift(req.shift)
1433+
contextShift = resolveContextShift(req.shift, req.model)
14181434
}
14191435
if runner.contextShift != contextShift {
14201436
return true

‎server/sched_test.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -907,16 +907,21 @@ func TestResolveContextShift(t *testing.T) {
907907
tests := []struct {
908908
name string
909909
shift *bool
910+
model *Model
910911
want bool
911912
}{
912913
{name: "unset defaults to shift", want: true},
914+
{name: "unset deepseek2 disables shift", model: &Model{Config: model.ConfigV2{ModelFamily: "deepseek2"}}, want: false},
915+
{name: "unset deepseek2 family disables shift", model: &Model{Config: model.ConfigV2{ModelFamilies: []string{"llama", "deepseek2"}}}, want: false},
913916
{name: "explicit false disables shift", shift: &falseValue, want: false},
917+
{name: "explicit false disables shift for deepseek2", shift: &falseValue, model: &Model{Config: model.ConfigV2{ModelFamily: "deepseek2"}}, want: false},
914918
{name: "explicit true enables shift", shift: &trueValue, want: true},
919+
{name: "explicit true enables shift for deepseek2", shift: &trueValue, model: &Model{Config: model.ConfigV2{ModelFamily: "deepseek2"}}, want: true},
915920
}
916921

917922
for _, tt := range tests {
918923
t.Run(tt.name, func(t *testing.T) {
919-
require.Equal(t, tt.want, resolveContextShift(tt.shift))
924+
require.Equal(t, tt.want, resolveContextShift(tt.shift, tt.model))
920925
})
921926
}
922927
}

0 commit comments

Comments
 (0)