Skip to content

Commit 0dc16b2

Browse files
emilymyeemilymye
committed
fix python rc release scripts
1 parent f94a00f commit 0dc16b2

3 files changed

Lines changed: 41 additions & 8 deletions

File tree

‎release/src/main/python-release/python_release_automation_utils.sh‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ function cleanup_pubsub() {
228228
# $2 - pid: the pid of running pipeline
229229
# $3 - running_job (DataflowRunner only): the job id of streaming pipeline running on DataflowRunner
230230
#######################################
231-
function verify_steaming_result() {
231+
function verify_streaming_result() {
232232
retry=3
233233
should_see="Python: "
234234
while(( $retry > 0 )); do
@@ -295,9 +295,11 @@ function verify_user_score() {
295295
function verify_hourly_team_score() {
296296
retry=3
297297
should_see='AntiqueBrassPlatypus'
298+
runner=$1
299+
298300
while(( $retry >= 0 )); do
299301
if [[ $retry > 0 ]]; then
300-
bq_pull_result=$(bq head -n 500 $DATASET.hourly_team_score_python_$1)
302+
bq_pull_result=$(bq head -n 500 ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner})
301303
if [[ $bq_pull_result = *"$should_see"* ]]; then
302304
echo "SUCCEED: hourly_team_score example successful run on $1-runner"
303305
break
@@ -307,14 +309,38 @@ function verify_hourly_team_score() {
307309
sleep 15
308310
fi
309311
else
310-
echo "FAILED: HourlyTeamScore example failed running on $1-runner. \
311-
Did not found scores of team $should_see in $DATASET.leader_board"
312+
echo "FAILED: HourlyTeamScore example failed running on $runner runner. \
313+
Did not found scores of team $should_see in ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"
312314
complete "FAILED"
313315
exit 1
314316
fi
315317
done
316318
}
317319

320+
function cleanup_hourly_team_score() {
321+
retry=3
322+
runner=$1
323+
324+
echo "Removing previously created table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}..."
325+
bq rm -q -f -t "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"
326+
327+
while(( $retry >= 0 )); do
328+
if [[ $retry > 0 ]]; then
329+
bq_ls_result=$(bq ls $DATASET)
330+
if [[ $bq_ls_result = *"${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}"* ]]; then
331+
retry=$(($retry-1))
332+
echo "${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} not cleaned up yet, waiting"
333+
sleep 1000
334+
else
335+
echo "Confirmed ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner} removed before running new test."
336+
break
337+
fi
338+
else
339+
echo "WARNING: Unable to clean up table ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}. \
340+
You may need to manually run 'bq rm -r -t ${DATASET}.${HOURLY_TEAM_SCORE_TABLE_PREFIX}_${runner}'."
341+
fi
342+
done
343+
}
318344

319345
# Python RC configurations
320346
VERSION=$(get_version)
@@ -336,4 +362,5 @@ PUBSUB_SUBSCRIPTION='wordstream-python-sub2'
336362
# Mobile Gaming Configurations
337363
DATASET='beam_postrelease_mobile_gaming'
338364
USERSCORE_OUTPUT_PREFIX='python-userscore_result'
365+
HOURLY_TEAM_SCORE_TABLE_PREFIX='hourly_team_score_python'
339366
GAME_INPUT_DATA='gs://dataflow-samples/game/5000_gaming_data.csv'

‎release/src/main/python-release/run_release_candidate_python_mobile_gaming.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,15 @@ function verify_userscore_dataflow() {
100100
#######################################
101101
function verify_hourlyteamscore_direct() {
102102
print_separator "Running HourlyTeamScore example with DirectRunner"
103+
# Clean up old bq tables
104+
cleanup_hourly_team_score "direct"
105+
103106
python -m apache_beam.examples.complete.game.hourly_team_score \
104107
--project=$PROJECT_ID \
105108
--dataset=$DATASET \
106109
--input=$GAME_INPUT_DATA \
107110
--temp_location=gs://$BUCKET_NAME/temp/ \
108-
--table="hourly_team_score_python_direct"
111+
--table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_direct"
109112

110113
verify_hourly_team_score "direct"
111114
}
@@ -121,6 +124,9 @@ function verify_hourlyteamscore_direct() {
121124
#######################################
122125
function verify_hourlyteamscore_dataflow() {
123126
print_separator "Running HourlyTeamScore example with DataflowRunner"
127+
# Clean up old bq tables
128+
cleanup_hourly_team_score "dataflow"
129+
124130
python -m apache_beam.examples.complete.game.hourly_team_score \
125131
--project=$PROJECT_ID \
126132
--region=$REGION_ID \
@@ -129,7 +135,7 @@ function verify_hourlyteamscore_dataflow() {
129135
--temp_location=gs://$BUCKET_NAME/temp/ \
130136
--sdk_location $BEAM_PYTHON_SDK \
131137
--input=$GAME_INPUT_DATA \
132-
--table="hourly_team_score_python_dataflow"
138+
--table="${HOURLY_TEAM_SCORE_TABLE_PREFIX}_dataflow"
133139

134140
verify_hourly_team_score "dataflow"
135141
}

‎release/src/main/python-release/run_release_candidate_python_quickstart.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ function verify_streaming_wordcount_direct() {
155155

156156
# verify result
157157
run_pubsub_publish
158-
verify_steaming_result "DirectRunner" $pid
158+
verify_streaming_result "DirectRunner" $pid
159159

160160
kill -9 $pid
161161
sleep 10
@@ -194,7 +194,7 @@ function verify_streaming_wordcount_dataflow() {
194194
# verify result
195195
run_pubsub_publish
196196
sleep 420
197-
verify_steaming_result "DataflowRunner" $pid $running_job
197+
verify_streaming_result "DataflowRunner" $pid $running_job
198198

199199
kill -9 $pid
200200
gcloud dataflow jobs cancel $running_job

0 commit comments

Comments
 (0)