Skip to content

Commit 412a6c7

Browse files
CAOShurongFernando Jesús Fernández Gallardo
authored andcommitted
sync: add pattern-based transfer ordering
Allow --order-by to rank files using comma-separated rclone path globs. Patterns are evaluated in order, unmatched files are placed last, and path ordering makes ties deterministic. Fixes rclone#3975
1 parent 9978dbc commit 412a6c7

3 files changed

Lines changed: 81 additions & 4 deletions

File tree

‎docs/content/docs.md‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2378,8 +2378,10 @@ describes what aspect is being measured:
23782378
- `size` - order by the size of the files
23792379
- `name` - order by the full path of the files
23802380
- `modtime` - order by the modification date of the files
2381+
- `pattern` - order by a list of path glob patterns
23812382

2382-
This can have a modifier appended with a comma:
2383+
The `size`, `name` and `modtime` comparisons can have a modifier appended with
2384+
a comma:
23832385

23842386
- `ascending` or `asc` - order so that the smallest (or oldest) is processed first
23852387
- `descending` or `desc` - order so that the largest (or newest) is processed first
@@ -2398,11 +2400,18 @@ processed continuously.
23982400

23992401
If no modifier is supplied then the order is `ascending`.
24002402

2403+
For `pattern`, append one or more comma-separated patterns. Patterns use the
2404+
[filter pattern syntax](/filtering/#patterns). Files matching the first pattern
2405+
are processed first, followed by files matching the second pattern, and so on.
2406+
Files which do not match a pattern are processed last. Within each group, files
2407+
are ordered by full path. Patterns cannot contain a comma.
2408+
24012409
For example
24022410

24032411
- `--order-by size,desc` - send the largest files first
24042412
- `--order-by modtime,ascending` - send the oldest files first
24052413
- `--order-by name` - send the files with alphabetically by path first
2414+
- `--order-by 'pattern,*.tar.gz,*.md5'` - send archives before checksum files
24062415

24072416
If the `--order-by` flag is not supplied or it is supplied with an
24082417
empty string then the default ordering will be used which is as

‎fs/sync/pipe.go‎

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010

1111
"github.com/aalpar/deheap"
1212
"github.com/rclone/rclone/fs"
13+
"github.com/rclone/rclone/fs/filter"
1314
"github.com/rclone/rclone/fs/fserrors"
1415
)
1516

@@ -187,8 +188,8 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
187188
if orderBy == "" {
188189
return nil, fraction, nil
189190
}
190-
parts := strings.Split(strings.ToLower(orderBy), ",")
191-
switch parts[0] {
191+
parts := strings.Split(orderBy, ",")
192+
switch strings.ToLower(parts[0]) {
192193
case "name":
193194
less = func(a, b fs.ObjectPair) bool {
194195
return a.Src.Remote() < b.Src.Remote()
@@ -202,12 +203,44 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
202203
ctx := context.Background()
203204
return a.Src.ModTime(ctx).Before(b.Src.ModTime(ctx))
204205
}
206+
case "pattern":
207+
if len(parts) < 2 {
208+
return nil, fraction, fmt.Errorf("pattern requires at least one glob")
209+
}
210+
matches := make([]func(string) bool, 0, len(parts)-1)
211+
for _, glob := range parts[1:] {
212+
if glob == "" {
213+
return nil, fraction, fmt.Errorf("empty pattern glob")
214+
}
215+
re, err := filter.GlobPathToRegexp(glob, false)
216+
if err != nil {
217+
return nil, fraction, fmt.Errorf("bad pattern glob %q: %w", glob, err)
218+
}
219+
matches = append(matches, re.MatchString)
220+
}
221+
rank := func(remote string) int {
222+
for i, match := range matches {
223+
if match(remote) {
224+
return i
225+
}
226+
}
227+
return len(matches)
228+
}
229+
less = func(a, b fs.ObjectPair) bool {
230+
aRemote, bRemote := a.Src.Remote(), b.Src.Remote()
231+
aRank, bRank := rank(aRemote), rank(bRemote)
232+
if aRank != bRank {
233+
return aRank < bRank
234+
}
235+
return aRemote < bRemote
236+
}
237+
return less, fraction, nil
205238
default:
206239
return nil, fraction, fmt.Errorf("unknown --order-by comparison %q", parts[0])
207240
}
208241
descending := false
209242
if len(parts) > 1 {
210-
switch parts[1] {
243+
switch strings.ToLower(parts[1]) {
211244
case "ascending", "asc":
212245
case "descending", "desc":
213246
descending = true

‎fs/sync/pipe_test.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,41 @@ func TestNewLess(t *testing.T) {
250250
assert.Contains(t, err.Error(), "unknown --order-by sort direction")
251251
})
252252

253+
t.Run("patternNeedsGlob", func(t *testing.T) {
254+
_, _, err := newLess("pattern")
255+
require.Error(t, err)
256+
assert.Contains(t, err.Error(), "pattern requires at least one glob")
257+
})
258+
259+
t.Run("patternRejectsEmptyGlob", func(t *testing.T) {
260+
_, _, err := newLess("pattern,")
261+
require.Error(t, err)
262+
assert.Contains(t, err.Error(), "empty pattern glob")
263+
})
264+
265+
t.Run("patternRejectsBadGlob", func(t *testing.T) {
266+
_, _, err := newLess("pattern,[")
267+
require.Error(t, err)
268+
assert.Contains(t, err.Error(), "bad pattern glob")
269+
})
270+
271+
t.Run("patternOrder", func(t *testing.T) {
272+
less, fraction, err := newLess("pattern,*.tar.gz,*.md5")
273+
require.NoError(t, err)
274+
assert.Equal(t, -1, fraction)
275+
276+
archiveA := fs.ObjectPair{Src: mockobject.New("a/archive.tar.gz")}
277+
archiveB := fs.ObjectPair{Src: mockobject.New("b/archive.tar.gz")}
278+
checksum := fs.ObjectPair{Src: mockobject.New("a/archive.md5")}
279+
unmatched := fs.ObjectPair{Src: mockobject.New("a/README")}
280+
281+
assert.True(t, less(archiveA, checksum))
282+
assert.True(t, less(checksum, unmatched))
283+
assert.False(t, less(unmatched, archiveA))
284+
assert.True(t, less(archiveA, archiveB))
285+
assert.False(t, less(archiveB, archiveA))
286+
})
287+
253288
var (
254289
obj1 = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone)
255290
obj2 = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone)

0 commit comments

Comments
 (0)