$ docker service create --name t2 --ulimit=nofile=100 debian /bin/bash -c "ulimit -a && sleep 30"
$ docker service logs -f t2
...
t2.1.dkm9duj5i4rq@aker    | open files                      (-n) 100
t2.1.dkm9duj5i4rq@aker    | real-time priority              (-r) 0
...

$ docker service update --ulimit-rm nofile --ulimit-add rtprio=1 t2
$ docker service logs -f t2
...
t2.1.dkm9duj5i4rq@aker    | open files                      (-n) 1048576
t2.1.dkm9duj5i4rq@aker    | real-time priority              (-r) 1
...

$ docker service update --ulimit-add nofile=100:200 t2
$ docker service inspect --pretty t2
...
Ulimits:
 nofile: 100:200
 nproc: -1:-1
...

$ docker service inspect t2
...
        "Ulimits": [
            {
                "Name": "nproc",
                "Hard": -1,
                "Soft": -1
            },
            {
                "Name": "nofile",
                "Hard": 200,
                "Soft": 100
            }
        ]
...

$ cat docker-compose.yaml
version: "3"

services:
  test:
    image: debian
    command: /bin/bash -c "ulimit -a && sleep 15"
    ulimits:
      nofile: 100
    deploy:
      mode: replicated

$ docker stack deploy --compose-file docker-compose.yaml test
$ docker service logs -f test_test | grep "open files"
test_test.1.jezlw4wt08rz@aker    | open files                      (-n) 100
$ docker inspect $(docker ps --filter=name=test_test -q)
...
        "Ulimits": [
            {
                "Name": "nofile",
                "Hard": 100,
                "Soft": 100
            }
        ],
...