Skip to content

Commit ddaf65a

Browse files
bsmith89bsmith89
committed
Merge branch 'main' into feat-persist-file-inventory
2 parents 7a41e63 + 8e95a12 commit ddaf65a

File tree

179 files changed

+1771
-4293
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

179 files changed

+1771
-4293
lines changed

‎.gitattributes‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,6 @@ tests/test_issue1046/expected-results/test_report.html linguist-vendored=true
66
tests/test_report/report.html linguist-vendored=true
77
tests/test_report/expected-results/report.html linguist-vendored=true
88
tests/test_report/custom-stylesheet.css linguist-vendored=true
9-
tests/**/*.html linguist-vendored=true
9+
# SCM syntax highlighting & preventing 3-way merges
10+
pixi.lock merge=binary linguist-language=YAML linguist-generated=true
11+
tests/**/*.html linguist-vendored=true

‎.github/workflows/docs.yml‎

Lines changed: 21 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,49 +17,45 @@ jobs:
1717
runs-on: ubuntu-latest
1818
steps:
1919
- uses: actions/checkout@v4
20-
21-
- uses: actions/setup-python@v4
22-
with:
23-
python-version: '3.x'
2420

25-
- name: Setup doc dependencies
26-
run: |
27-
python -m pip install --upgrade pip
28-
pip install -r docs/requirements.txt
21+
- name: Install Pixi
22+
uses: prefix-dev/[email protected]
23+
with:
24+
environments: "docs"
25+
pixi-version: v0.42.1
26+
cache: false # no pixi.lock
27+
locked: false
2928

30-
- name: Build docs
29+
- name: Build Docs
3130
run: |
32-
cd docs
33-
make html SPHINXOPTS="-W --keep-going -n"
31+
pixi run --environment docs build-docs
32+
3433
- name: Upload documentation
3534
uses: actions/upload-artifact@v4
3635
with:
3736
name: docs-html
3837
path: docs/_build/html/
39-
38+
4039
apidocs:
4140
if: github.event.pull_request.merged != true || github.ref != 'refs/heads/main'
4241
runs-on: ubuntu-latest
4342
steps:
4443
- uses: actions/checkout@v4
45-
46-
- uses: actions/setup-python@v4
44+
45+
- name: Install Pixi
46+
uses: prefix-dev/[email protected]
4747
with:
48-
python-version: '3.x'
48+
environments: "docs"
49+
pixi-version: v0.42.1
50+
cache: false # no pixi.lock
51+
locked: false
4952

50-
- name: Setup doc dependencies
51-
run: |
52-
python -m pip install --upgrade pip
53-
pip install -r apidocs/requirements.txt
54-
pip install -e .
55-
56-
- name: Build docs
53+
- name: Build API Docs
5754
run: |
58-
cd apidocs
59-
make html SPHINXOPTS="--keep-going -n"
55+
pixi run --environment docs build-apidocs
6056
6157
- name: Upload documentation
6258
uses: actions/upload-artifact@v4
6359
with:
6460
name: apidocs-html
65-
path: apidocs/_build/html/
61+
path: apidocs/_build/html/

‎.github/workflows/main.yml‎

Lines changed: 55 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,16 @@ jobs:
2121
steps:
2222
- uses: actions/checkout@v4
2323

24-
- uses: mamba-org/setup-micromamba@v1
24+
- name: Install Pixi
25+
uses: prefix-dev/[email protected]
2526
with:
26-
environment-name: black
27-
create-args: black
28-
cache-environment: false
27+
environments: "quality"
28+
pixi-version: v0.42.1
29+
cache: false # no pixi.lock
30+
locked: false
2931

30-
- name: Check formatting
31-
shell: bash -el {0}
32-
run: black --check --diff .
32+
- name: Run black
33+
run: pixi run -e quality format --check --diff .
3334

3435
- name: Comment PR
3536
if: github.event_name == 'pull_request' && failure()
@@ -38,181 +39,103 @@ jobs:
3839
message: "Please format your code with [black](https://black.readthedocs.io): `black snakemake tests/*.py`."
3940
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4041

41-
testing:
42+
################################################################################################
43+
# Testing: Run tests using Pixi
44+
################################################################################################
45+
tests:
4246
if: github.event.pull_request.merged != true || github.ref != 'refs/heads/main'
4347
strategy:
48+
fail-fast: false
4449
matrix:
4550
test_group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
46-
py_ver:
47-
- "3.11"
48-
- "3.12"
51+
# see pyprojec.toml: [tool.pixi.feature.test] for available test types
52+
os: [ubuntu-latest, windows-latest] # macos-latest, macos-13 not supported yet
53+
env: ["py311", "py312"]
4954
shell:
5055
- bash
51-
#- dash # for py 3.12 also test dash as default shell (snakemake will still use bash for shell commands)
52-
exclude: []
53-
#- shell: dash
54-
# py_ver: "3.11"
55-
runs-on: ubuntu-latest
56-
needs: formatting
56+
57+
runs-on: ${{ matrix.os }}
58+
5759
env:
5860
AWS_AVAILABLE: "${{ secrets.AWS_ACCESS_KEY_ID }}"
5961
GCP_AVAILABLE: "${{ secrets.GCP_SA_KEY }}"
62+
ZENODO_SANDBOX_PAT: "${{ secrets.ZENODO_SANDBOX_PAT }}"
63+
CI: true
64+
6065
steps:
6166
- uses: actions/checkout@v4
6267
with:
6368
fetch-depth: 0
6469

65-
- name: Modify python version in test env
66-
shell: bash -el {0}
67-
run: |
68-
cp test-environment.yml test-environment-${{ matrix.py_ver }}.yml
69-
sed -E -i 's/- python.+/- python =${{ matrix.py_ver }}/' test-environment-${{ matrix.py_ver }}.yml
70-
71-
- name: Setup snakemake environment
72-
uses: conda-incubator/setup-miniconda@v3
70+
- name: Install Pixi
71+
uses: prefix-dev/[email protected]
7372
with:
74-
miniforge-version: latest
75-
environment-file: test-environment-${{ matrix.py_ver }}.yml
76-
environment-name: snakemake
77-
auto-update-conda: true
78-
79-
- name: Install snakemake from source
80-
shell: bash -el {0}
81-
run: |
82-
conda config --set channel_priority strict
83-
84-
pip install -e .
85-
86-
# - name: Configure fast APT mirror
87-
# uses: vegardit/[email protected]
88-
89-
- name: Setup apt dependencies
90-
run: |
91-
sudo add-apt-repository -y ppa:apptainer/ppa
92-
sudo apt-get update
93-
sudo apt install -y git wget openmpi-bin libopenmpi-dev apptainer dash
73+
environments: ${{ matrix.env }}
74+
pixi-version: v0.42.1
75+
cache: false # no pixi.lock
76+
locked: false
9477

9578
# See https://github.com/apptainer/apptainer/pull/2262
96-
- name: Disable apparmor namespace restrictions for apptainer
79+
- name: Disable apparmor namespace restrictions for apptainer (os='Linux')
80+
if: runner.os == 'Linux'
9781
run: |
9882
sudo sh -c 'echo kernel.apparmor_restrict_unprivileged_userns=0 \
9983
>/etc/sysctl.d/90-disable-userns-restrictions.conf'
10084
sudo sysctl -p /etc/sysctl.d/90-disable-userns-restrictions.conf
10185
102-
- name: Setup minio
86+
- name: Setup MinIO for AWS S3 testing (os='Linux', for AWS S3 testing)
87+
if: runner.os == 'Linux'
10388
uses: comfuture/minio-action@v1
10489
with:
10590
access_key: minio
10691
secret_key: minio123
10792
port: 9000
10893

109-
- name: Test minio
94+
- name: Test MinIO (os='Linux')
95+
if: runner.os == 'Linux'
11096
run: |
11197
export AWS_ACCESS_KEY_ID=minio
11298
export AWS_SECRET_ACCESS_KEY=minio123
11399
export AWS_EC2_METADATA_DISABLED=true
114100
aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://test
115101
116-
- name: Test local
117-
env:
118-
CI: true
119-
ZENODO_SANDBOX_PAT: "${{ secrets.ZENODO_SANDBOX_PAT }}"
120-
#PYTHONTRACEMALLOC: 10
121-
shell: bash -el {0}
102+
- name: Run Tests for Linux (os= 'Linux')
103+
if: runner.os == 'Linux'
122104
run: |
123-
if [ "${{ matrix.shell }}" == "dash" ]; then
124-
sudo chsh -s /usr/bin/dash $(whoami)
125-
fi
126-
127-
pytest -v -x --show-capture=stderr \
128-
--splits 10 --group ${{ matrix.test_group }} --splitting-algorithm=least_duration \
129-
tests/tests.py \
130-
tests/tests_using_conda.py \
131-
tests/test_expand.py \
132-
tests/test_io.py \
133-
tests/test_schema.py \
134-
tests/test_linting.py \
135-
tests/test_executor_test_suite.py \
136-
tests/test_api.py \
137-
tests/test_internals.py
138-
139-
- name: Run tests/test_report/Snakefile to generate report
140-
shell: bash -el {0}
105+
pixi run --environment ${{matrix.env}} test-all \
106+
--splits 10 \
107+
--group ${{ matrix.test_group }} \
108+
--splitting-algorithm=least_duration
109+
110+
- name: Run Tests for Windows (os='Windows')
111+
if: runner.os == 'Windows'
141112
run: |
142-
cd tests/test_report
143-
snakemake --use-conda --cores 1 --report report.zip
113+
pixi run --environment ${{matrix.env}} test-simple --splits 10 --group ${{ matrix.test_group }} --splitting-algorithm=least_duration
144114
145-
- name: List directory structure
115+
- name: Run Report Generation Tests (os='Linux')
116+
if: runner.os == 'Linux'
117+
shell: bash -el {0}
146118
run: |
147-
ls -R
148-
149-
- name: Upload report
150-
if: ${{ matrix.test_group == 1 && matrix.shell == 'dash'}}
151-
uses: actions/upload-artifact@v4
152-
with:
153-
name: report-${{ matrix.py_ver }}.zip
154-
path: tests/test_report/report.zip
119+
cd tests/test_report
120+
pixi run -e ${{ matrix.env }} snakemake \
121+
--use-conda \
122+
--cores 1 \
123+
--report report.zip
155124
156125
build-container-image:
157126
if: github.event.pull_request.merged != true || github.ref != 'refs/heads/main'
158127
runs-on: ubuntu-latest
159-
needs: testing
128+
needs: tests
160129
steps:
161130
- uses: actions/checkout@v4
162131

163132
- name: Build container image
164133
run: docker build .
165134

166-
testing-windows:
167-
if: github.event.pull_request.merged != true || github.ref != 'refs/heads/main'
168-
strategy:
169-
matrix:
170-
test_group: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
171-
runs-on: windows-latest
172-
needs: formatting
173-
steps:
174-
- uses: actions/checkout@v4
175-
with:
176-
fetch-depth: 0
177-
- name: Remove unix-only dependencies
178-
shell: python
179-
run: |
180-
import fileinput
181-
excluded_on_win = [ "environment-modules",
182-
"cwltool",
183-
"cwl-utils",
184-
"apptainer",
185-
"squashfuse",
186-
"stress-ng" ]
187-
for line in fileinput.input("test-environment.yml", inplace=True):
188-
if all(pkg not in line for pkg in excluded_on_win):
189-
print(line, end="")
190-
191-
- name: Setup snakemake environment
192-
uses: conda-incubator/setup-miniconda@v3
193-
with:
194-
miniforge-version: latest
195-
environment-file: test-environment.yml
196-
environment-name: snakemake
197-
auto-update-conda: true
198-
199-
- name: Install snakemake from source
200-
run: |
201-
202-
pip install .
203-
204-
- name: Run tests
205-
env:
206-
CI: true
207-
ZENODO_SANDBOX_PAT: "${{ secrets.ZENODO_SANDBOX_PAT }}"
208-
run: |
209-
python -m pytest --show-capture=stderr -v -x --splits 10 --group ${{ matrix.test_group }} --splitting-algorithm=least_duration tests/tests.py
210-
211135
testing-done:
212136
if: github.event.pull_request.merged != true || github.ref != 'refs/heads/main'
213137
needs:
214-
- testing
215-
- testing-windows
138+
- tests
216139
runs-on: ubuntu-latest
217140
steps:
218-
- run: echo "All tests passed."
141+
- run: echo "All tests passed."

‎.gitignore‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,10 @@ tests/test*/*
3030
playground/*
3131
tutorial/*
3232

33-
snakemake/assets/data
33+
src/snakemake/assets/data
34+
src/snakemake/_version.py
35+
36+
# pixi environments
37+
.pixi
38+
*.egg-info
39+
pixi.lock

‎MANIFEST.in‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
1-
include versioneer.py
2-
include snakemake/_version.py
1+
global-exclude *
2+
3+
include setup.py
4+
include pyproject.toml
5+
include README.md
36
include LICENSE.md
7+
include .gitignore
8+
graft src
49

510
# Include tests in sdist, otherwise only tests/test*.py are included by default
611
graft tests
12+
13+
global-exclude *~ *.py[cod] *.so

‎apidocs/requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,5 @@ appdirs
99
sphinxawesome-theme
1010
snakemake-interface-common
1111
snakemake-interface-executor-plugins
12-
snakemake-interface-storage-plugins
12+
snakemake-interface-storage-plugins
13+
snakemake-interface-logger-plugins

‎doc-environment.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ dependencies:
1919
- snakemake-interface-executor-plugins >=9.1.0
2020
- snakemake-interface-storage-plugins >=3.2.3
2121
- sphinx-argparse
22+
- sphinx-tabs
2223
- sphinx_rtd_theme
2324
- sphinxawesome-theme
2425
- docutils

0 commit comments

Comments
 (0)