Skip to content

Commit 564bea7

Browse files
liushilongbuaaliushilongbuaa
authored
[ci] Paralize azure pipeline (sonic-net#1040)
1. Setup pipeline without manual effort when checkout new release branch. 2. Use correct branch when downloading artifacts or checkout relative repos. 3. Clear downloaded artifacts to avoid using outdated dependencies. 4. Use commonlib pipeline to download libnl3 and libyang instead of vs image build, to increase success rate. 5. Add weekly build to keep artifacts remaining.
1 parent 57ed180 commit 564bea7

File tree

5 files changed

+137
-69
lines changed

5 files changed

+137
-69
lines changed

‎.azure-pipelines/build-docker-sonic-vs-template.yml‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,40 +32,48 @@ jobs:
3232
vmImage: 'ubuntu-20.04'
3333

3434
steps:
35+
- checkout: self
36+
clean: true
3537
- task: DownloadPipelineArtifact@2
3638
inputs:
3739
source: specific
3840
project: build
39-
pipeline: 9
41+
pipeline: Azure.sonic-swss-common
4042
artifact: ${{ parameters.swss_common_artifact_name }}
43+
path: $(Build.ArtifactStagingDirectory)/download
4144
runVersion: 'latestFromBranch'
42-
runBranch: 'refs/heads/master'
45+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
4346
displayName: "Download sonic swss common deb packages"
4447
- task: DownloadPipelineArtifact@2
4548
inputs:
4649
artifact: ${{ parameters.sairedis_artifact_name }}
50+
path: $(Build.ArtifactStagingDirectory)/download
4751
displayName: "Download sonic sairedis deb packages"
4852
- task: DownloadPipelineArtifact@2
4953
inputs:
5054
artifact: ${{ parameters.swss_artifact_name }}
55+
path: $(Build.ArtifactStagingDirectory)/download
5156
displayName: "Download sonic swss artifact"
5257
- task: DownloadPipelineArtifact@2
5358
inputs:
5459
source: specific
5560
project: build
56-
pipeline: 142
61+
pipeline: Azure.sonic-buildimage.official.vs
5762
artifact: sonic-buildimage.vs
5863
runVersion: 'latestFromBranch'
59-
runBranch: 'refs/heads/master'
60-
displayName: "Download sonic buildimage"
64+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
65+
path: $(Build.ArtifactStagingDirectory)/download
66+
patterns: '**/target/docker-sonic-vs.gz'
67+
displayName: "Download sonic-buildimage docker-sonic-vs"
6168
- script: |
69+
set -ex
6270
echo $(Build.DefinitionName).$(Build.BuildNumber)
6371
64-
docker load < ../target/docker-sonic-vs.gz
72+
docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
6573
6674
mkdir -p .azure-pipelines/docker-sonic-vs/debs
6775
68-
cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs
76+
cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
6977
7078
pushd .azure-pipelines
7179
@@ -74,7 +82,8 @@ jobs:
7482
popd
7583
7684
docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
77-
displayName: "Build sonic-docker-vs"
85+
rm -rf $(Build.ArtifactStagingDirectory)/download
86+
displayName: "Build docker-sonic-vs"
7887
7988
- publish: $(Build.ArtifactStagingDirectory)/
8089
artifact: ${{ parameters.artifact_name }}

‎.azure-pipelines/build-swss-template.yml‎

Lines changed: 49 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ parameters:
2121
- name: sonic_slave
2222
type: string
2323

24+
- name: debian_version
25+
type: string
26+
2427
- name: sairedis_artifact_name
2528
type: string
2629

@@ -38,13 +41,22 @@ jobs:
3841
pool:
3942
${{ if ne(parameters.pool, 'default') }}:
4043
name: ${{ parameters.pool }}
41-
${{ if eq(parameters.pool, 'default') }}:
44+
${{ else }}:
4245
vmImage: 'ubuntu-20.04'
4346

4447
container:
4548
image: sonicdev-microsoft.azurecr.io:443/${{ parameters.sonic_slave }}:latest
4649

4750
steps:
51+
- checkout: sonic-swss
52+
submodules: true
53+
clean: true
54+
- script: |
55+
set -ex
56+
git checkout $(BUILD_BRANCH)
57+
git submodule update
58+
git status
59+
displayName: Set up sonic-swss branch
4860
- script: |
4961
sudo apt-get install -y libhiredis0.14 libhiredis-dev
5062
sudo apt-get install -y libzmq5 libzmq3-dev
@@ -60,51 +72,57 @@ jobs:
6072
inputs:
6173
source: specific
6274
project: build
63-
pipeline: 9
75+
pipeline: Azure.sonic-swss-common
6476
artifact: ${{ parameters.swss_common_artifact_name }}
6577
runVersion: 'latestFromBranch'
66-
runBranch: 'refs/heads/master'
78+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
79+
path: $(Build.ArtifactStagingDirectory)/download
80+
patterns: |
81+
libswsscommon_1.0.0_*.deb
82+
libswsscommon-dev_1.0.0*.deb
6783
displayName: "Download sonic swss common deb packages"
6884
- task: DownloadPipelineArtifact@2
6985
inputs:
7086
artifact: ${{ parameters.sairedis_artifact_name }}
71-
displayName: "Download sonic sairedis deb packages"
87+
path: $(Build.ArtifactStagingDirectory)/download
88+
patterns: |
89+
libsaivs_*.deb
90+
libsaivs-dev_*.deb
91+
libsairedis_*.deb
92+
libsairedis-dev_*.deb
93+
libsaimetadata_*.deb
94+
libsaimetadata-dev_*.deb
95+
syncd-vs_*.deb
96+
displayName: "Download pre-stage built ${{ parameters.sairedis_artifact_name }}"
7297
- task: DownloadPipelineArtifact@2
7398
inputs:
7499
source: specific
75100
project: build
76-
pipeline: 1
77-
artifact: sonic-buildimage.vs
101+
pipeline: Azure.sonic-buildimage.common_libs
78102
runVersion: 'latestFromBranch'
79-
runBranch: 'refs/heads/master'
80-
displayName: "Download sonic buildimage"
103+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
104+
path: $(Build.ArtifactStagingDirectory)/download
105+
artifact: common-lib
106+
patterns: |
107+
target/debs/${{ parameters.debian_version }}/libnl-3*.deb
108+
target/debs/${{ parameters.debian_version }}/libnl-genl*.deb
109+
target/debs/${{ parameters.debian_version }}/libnl-route*.deb
110+
target/debs/${{ parameters.debian_version }}/libnl-nf*.deb
111+
displayName: "Download common libs"
112+
81113
- script: |
82-
sudo dpkg -i target/debs/buster/libnl-3-200_*.deb
83-
sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb
84-
sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb
85-
sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb
86-
sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb
87-
sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb
88-
sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb
89-
sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb
90-
sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
91-
sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
92-
sudo dpkg -i libsaivs_*.deb
93-
sudo dpkg -i libsaivs-dev_*.deb
94-
sudo dpkg -i libsairedis_*.deb
95-
sudo dpkg -i libsairedis-dev_*.deb
96-
sudo dpkg -i libsaimetadata_*.deb
97-
sudo dpkg -i libsaimetadata-dev_*.deb
98-
sudo dpkg -i syncd-vs_*.deb
99-
workingDirectory: $(Pipeline.Workspace)
114+
set -ex
115+
sudo dpkg -i $(find ./download -name *.deb)
116+
rm -rf download || true
117+
workingDirectory: $(Build.ArtifactStagingDirectory)
100118
displayName: "Install libnl3, sonic swss common, and sairedis"
101-
- checkout: sonic-swss
102-
path: s
103-
submodules: true
104119
- script: |
120+
set -ex
121+
rm ../*.deb || true
105122
./autogen.sh
106-
dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
123+
dpkg-buildpackage -us -uc -b -j$(nproc)
124+
mv ../*.deb $(Build.ArtifactStagingDirectory)
107125
displayName: "Compile sonic swss"
108-
- publish: $(System.DefaultWorkingDirectory)/
126+
- publish: $(Build.ArtifactStagingDirectory)
109127
artifact: ${{ parameters.artifact_name }}
110128
displayName: "Archive swss debian packages"

‎.azure-pipelines/build-template.yml‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ jobs:
5151
pool:
5252
${{ if ne(parameters.pool, 'default') }}:
5353
name: ${{ parameters.pool }}
54-
${{ if eq(parameters.pool, 'default') }}:
54+
${{ else }}:
5555
vmImage: 'ubuntu-20.04'
5656

5757
container:
@@ -97,22 +97,25 @@ jobs:
9797
inputs:
9898
source: specific
9999
project: build
100-
pipeline: 9
100+
pipeline: Azure.sonic-swss-common
101101
artifact: ${{ parameters.swss_common_artifact_name }}
102102
runVersion: 'latestFromBranch'
103-
runBranch: 'refs/heads/master'
104-
path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}'
103+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
104+
path: $(Build.ArtifactStagingDirectory)/download
105105
displayName: "Download sonic swss common deb packages"
106106
- script: |
107-
cd $(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}
108-
sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
109-
sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
110-
workingDirectory: $(Pipeline.Workspace)
107+
set -ex
108+
sudo dpkg -i download/libswsscommon_1.0.0_${{ parameters.arch }}.deb
109+
sudo dpkg -i download/libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
110+
rm -rf download
111+
workingDirectory: $(Build.ArtifactStagingDirectory)
111112
displayName: "Install sonic swss Common"
112113
- script: |
113114
set -ex
115+
rm ../*.deb || true
114116
./autogen.sh
115-
fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs && cp ../*.deb .
117+
fakeroot debian/rules DEB_CONFIGURE_EXTRA_FLAGS='--enable-code-coverage' CFLAGS="" CXXFLAGS="" binary-syncd-vs
118+
mv ../*.deb .
116119
displayName: "Compile sonic sairedis with coverage enabled"
117120
- script: |
118121
sudo cp azsyslog.conf /etc/rsyslog.conf

‎.azure-pipelines/test-docker-sonic-vs-template.yml‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,33 +18,39 @@ jobs:
1818
ls -A1 | xargs -I{} sudo rm -rf {}
1919
displayName: "Clean workspace"
2020
- checkout: self
21+
clean: true
22+
displayName: "Checkout sonic-sairedis"
23+
- checkout: sonic-swss
24+
clean: true
25+
displayName: "Checkout sonic-swss"
26+
- script: |
27+
set -ex
28+
cd sonic-swss
29+
git checkout $(BUILD_BRANCH)
30+
displayName: Set up sonic-swss branch
2131
- task: DownloadPipelineArtifact@2
2232
inputs:
2333
artifact: docker-sonic-vs
24-
displayName: "Download docker sonic vs image"
25-
34+
path: $(Build.ArtifactStagingDirectory)/download
35+
displayName: "Download pre-stage built docker-sonic-vs"
2636
- task: DownloadPipelineArtifact@2
2737
inputs:
2838
source: specific
2939
project: build
30-
pipeline: 9
40+
pipeline: Azure.sonic-swss-common
3141
artifact: sonic-swss-common.amd64.ubuntu20_04
42+
path: $(Build.ArtifactStagingDirectory)/download
3243
runVersion: 'latestFromBranch'
33-
runBranch: 'refs/heads/master'
44+
runBranch: 'refs/heads/$(BUILD_BRANCH)'
3445
displayName: "Download sonic swss common deb packages"
3546

36-
- checkout: self
37-
displayName: "Checkout sonic-sairedis"
38-
- checkout: sonic-swss
39-
displayName: "Checkout sonic-swss"
40-
4147
- script: |
42-
set -x
48+
set -ex
4349
sudo sonic-sairedis/.azure-pipelines/build_and_install_module.sh
4450
4551
sudo apt-get install -y libhiredis0.14
46-
sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
47-
sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
52+
sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
53+
sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
4854
4955
# install packages for vs test
5056
sudo apt-get install -y net-tools bridge-utils vlan
@@ -53,12 +59,13 @@ jobs:
5359
displayName: "Install dependencies"
5460
5561
- script: |
56-
set -x
57-
sudo docker load -i ../docker-sonic-vs.gz
62+
set -ex
63+
sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
5864
docker ps
5965
ip netns list
6066
pushd sonic-swss/tests
6167
sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
68+
rm -rf $(Build.ArtifactStagingDirectory)/download
6269
displayName: "Run vs tests"
6370
6471
- task: PublishTestResults@2

‎azure-pipelines.yml‎

Lines changed: 38 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,29 @@
33
# Add steps that build, run tests, deploy, and more:
44
# https://aka.ms/yaml
55

6+
pr:
7+
- master
8+
- 202???
9+
- 201???
10+
611
trigger:
12+
batch: true
713
branches:
814
include:
9-
- "*"
15+
- master
16+
- 202???
17+
- 201???
18+
19+
# this part need to be set in UI
20+
schedules:
21+
- cron: "0 0 * * 6"
22+
displayName: Weekly build
23+
branches:
24+
include:
25+
- master
26+
- 202???
27+
- 201???
28+
always: true
1029

1130
resources:
1231
repositories:
@@ -15,14 +34,25 @@ resources:
1534
name: Azure/sonic-swss
1635
endpoint: build
1736

37+
parameters:
38+
- name: debian_version
39+
type: string
40+
default: buster
41+
variables:
42+
- name: BUILD_BRANCH
43+
${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
44+
value: $(System.PullRequest.TargetBranch)
45+
${{ else }}:
46+
value: $(Build.SourceBranchName)
47+
1848
stages:
1949
- stage: Build
2050

2151
jobs:
2252
- template: .azure-pipelines/build-template.yml
2353
parameters:
2454
arch: amd64
25-
sonic_slave: sonic-slave-buster
55+
sonic_slave: sonic-slave-${{ parameters.debian_version }}
2656
swss_common_artifact_name: sonic-swss-common
2757
artifact_name: sonic-sairedis
2858
syslog_artifact_name: sonic-sairedis.syslog
@@ -36,19 +66,19 @@ stages:
3666
- template: .azure-pipelines/build-template.yml
3767
parameters:
3868
arch: armhf
39-
timeout: 240
69+
timeout: 180
4070
pool: sonicbld-armhf
41-
sonic_slave: sonic-slave-buster-armhf
71+
sonic_slave: sonic-slave-${{ parameters.debian_version }}-armhf
4272
swss_common_artifact_name: sonic-swss-common.armhf
4373
artifact_name: sonic-sairedis.armhf
4474
syslog_artifact_name: sonic-sairedis.syslog.armhf
4575

4676
- template: .azure-pipelines/build-template.yml
4777
parameters:
4878
arch: arm64
49-
timeout: 240
79+
timeout: 180
5080
pool: sonicbld-arm64
51-
sonic_slave: sonic-slave-buster-arm64
81+
sonic_slave: sonic-slave-${{ parameters.debian_version }}-arm64
5282
swss_common_artifact_name: sonic-swss-common.arm64
5383
artifact_name: sonic-sairedis.arm64
5484
syslog_artifact_name: sonic-sairedis.syslog.arm64
@@ -60,10 +90,11 @@ stages:
6090
- template: .azure-pipelines/build-swss-template.yml
6191
parameters:
6292
arch: amd64
63-
sonic_slave: sonic-slave-buster
93+
sonic_slave: sonic-slave-${{ parameters.debian_version }}
6494
swss_common_artifact_name: sonic-swss-common
6595
sairedis_artifact_name: sonic-sairedis
6696
artifact_name: sonic-swss
97+
debian_version: ${{ parameters.debian_version }}
6798

6899
- stage: BuildDocker
69100
dependsOn: BuildSwss

0 commit comments

Comments
 (0)