Skip to content

Commit fab085c

Browse files
author
MarcoFalke
committed
contrib: Use text=True in subprocess over manual encoding handling
All touched Python scripts already assume and require UTF8, so manually specifying encoding or decoding for functions in the subprocess module is redundant to just using text=True, which exists since Python 3.7
1 parent fa71c15 commit fab085c

File tree

12 files changed

+43
-47
lines changed

12 files changed

+43
-47
lines changed

‎contrib/devtools/copyright_header.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,12 @@ def applies_to_file(filename):
5454
GIT_TOPLEVEL_CMD = 'git rev-parse --show-toplevel'.split(' ')
5555

5656
def call_git_ls(base_directory):
57-
out = subprocess.check_output([*GIT_LS_CMD, base_directory])
58-
return [f for f in out.decode("utf-8").split('\n') if f != '']
57+
out = subprocess.check_output([*GIT_LS_CMD, base_directory], text=True)
58+
return [f for f in out.split('\n') if f != '']
5959

6060
def call_git_toplevel():
6161
"Returns the absolute path to the project root"
62-
return subprocess.check_output(GIT_TOPLEVEL_CMD).strip().decode("utf-8")
62+
return subprocess.check_output(GIT_TOPLEVEL_CMD, text=True).strip()
6363

6464
def get_filenames_to_examine(base_directory):
6565
"Returns an array of absolute paths to any project files in the base_directory that pass the include/exclude filters"
@@ -298,8 +298,8 @@ def report_cmd(argv):
298298
GIT_LOG_CMD = "git log --pretty=format:%%ai %s"
299299

300300
def call_git_log(filename):
301-
out = subprocess.check_output((GIT_LOG_CMD % filename).split(' '))
302-
return out.decode("utf-8").split('\n')
301+
out = subprocess.check_output((GIT_LOG_CMD % filename).split(' '), text=True)
302+
return out.split('\n')
303303

304304
def get_git_change_years(filename):
305305
git_log_lines = call_git_log(filename)

‎contrib/verify-commits/verify-commits.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,8 @@ def main():
140140

141141
# Check that the commit (and parents) was signed with a trusted key
142142
valid_sig = False
143-
verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], capture_output=True)
144-
for line in verify_res.stderr.decode().splitlines():
143+
verify_res = subprocess.run([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', "--raw", current_commit], text=True, capture_output=True)
144+
for line in verify_res.stderr.splitlines():
145145
if line.startswith("[GNUPG:] VALIDSIG "):
146146
key = line.split(" ")[-1]
147147
valid_sig = key in trusted_keys
@@ -152,7 +152,7 @@ def main():
152152
if prev_commit != "":
153153
print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
154154
print("Parents are:", file=sys.stderr)
155-
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit]).decode('utf8').splitlines()[0].split(' ')
155+
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], text=True).splitlines()[0].split(' ')
156156
for parent in parents:
157157
subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
158158
else:
@@ -162,29 +162,29 @@ def main():
162162
# Check the Tree-SHA512
163163
if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
164164
tree_hash = tree_sha512sum(current_commit)
165-
if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit]).decode('utf8').splitlines():
165+
if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], text=True).splitlines():
166166
print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
167167
sys.exit(1)
168168

169169
# Merge commits should only have two parents
170-
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit]).decode('utf8').splitlines()[0].split(' ')
170+
parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], text=True).splitlines()[0].split(' ')
171171
if len(parents) > 2:
172172
print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
173173
sys.exit(1)
174174

175175
# Check that the merge commit is clean
176-
commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit]).decode('utf8').splitlines()[0])
176+
commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], text=True).splitlines()[0])
177177
check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60 # Only check commits in clean_merge days
178178
allow_unclean = current_commit in unclean_merge_allowed
179179
if len(parents) == 2 and check_merge and not allow_unclean:
180-
current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0]
180+
current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], text=True).splitlines()[0]
181181

182182
# This merge-tree functionality requires git >= 2.38. The
183183
# --write-tree option was added in order to opt-in to the new
184184
# behavior. Older versions of git will not recognize the option and
185185
# will instead exit with code 128.
186186
try:
187-
recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]]).decode('utf8').splitlines()[0]
187+
recreated_tree = subprocess.check_output([GIT, "merge-tree", "--write-tree", parents[0], parents[1]], text=True).splitlines()[0]
188188
except subprocess.CalledProcessError as e:
189189
if e.returncode == 128:
190190
print("git v2.38+ is required for this functionality.", file=sys.stderr)

‎test/functional/feature_framework_startup_failures.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,13 @@ def _verify_startup_failure(self, test, internal_args, expected_exception):
4343
args = [sys.executable] + sys.argv + [f"--tmpdir={self.options.tmpdir}/{subdir}", f"--internal_test={test.__name__}"] + internal_args
4444

4545
try:
46-
# The subprocess encoding argument gives different results for e.output
47-
# on Linux/Windows, so handle decoding by ourselves for consistency.
4846
output = subprocess.run(args, timeout=60 * self.options.timeout_factor,
49-
stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode("utf-8")
47+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True).stdout
5048
except subprocess.TimeoutExpired as e:
5149
print("Unexpected child process timeout!\n"
5250
"WARNING: Timeouts like this halt execution of TestNode logic, "
5351
"meaning dangling bitcoind processes are to be expected.\n"
54-
f"<CHILD OUTPUT BEGIN>\n{e.output.decode('utf-8')}\n<CHILD OUTPUT END>",
52+
f"<CHILD OUTPUT BEGIN>\n{e.output}\n<CHILD OUTPUT END>",
5553
file=sys.stderr)
5654
raise
5755

‎test/functional/tool_signet_miner.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/env python3
2-
# Copyright (c) 2022 The Bitcoin Core developers
2+
# Copyright (c) 2022-present The Bitcoin Core developers
33
# Distributed under the MIT software license, see the accompanying
44
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
55
"""Test signet miner tool"""
@@ -102,8 +102,8 @@ def mine_block_manual(self, node, *, sign):
102102
'genpsbt',
103103
f'--address={node.getnewaddress()}',
104104
'--poolnum=98',
105-
], check=True, input=json.dumps(template).encode('utf8'), capture_output=True)
106-
psbt = genpsbt.stdout.decode('utf8').strip()
105+
], check=True, text=True, input=json.dumps(template), capture_output=True)
106+
psbt = genpsbt.stdout.strip()
107107
if sign:
108108
self.log.debug("Sign the PSBT")
109109
res = node.walletprocesspsbt(psbt=psbt, sign=True, sighashtype='ALL')
@@ -112,8 +112,8 @@ def mine_block_manual(self, node, *, sign):
112112
solvepsbt = subprocess.run(base_cmd + [
113113
'solvepsbt',
114114
f'--grind-cmd={shlex.join(util_argv)}',
115-
], check=True, input=psbt.encode('utf8'), capture_output=True)
116-
node.submitblock(solvepsbt.stdout.decode('utf8').strip())
115+
], check=True, text=True, input=psbt, capture_output=True)
116+
node.submitblock(solvepsbt.stdout.strip())
117117
assert_equal(node.getblockcount(), n_blocks + 1)
118118

119119
def run_test(self):

‎test/lint/check-doc.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
#!/usr/bin/env python3
2-
# Copyright (c) 2015-2022 The Bitcoin Core developers
2+
# Copyright (c) 2015-present The Bitcoin Core developers
33
# Distributed under the MIT software license, see the accompanying
44
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
55

66
'''
77
This checks if all command line args are documented.
88
Return value is 0 to indicate no error.
9-
10-
Author: @MarcoFalke
119
'''
1210

1311
from subprocess import check_output
@@ -27,8 +25,8 @@
2725

2826

2927
def lint_missing_argument_documentation():
30-
used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip()
31-
docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip()
28+
used = check_output(CMD_GREP_ARGS, shell=True, text=True).strip()
29+
docd = check_output(CMD_GREP_DOCS, shell=True, text=True).strip()
3230

3331
args_used = set(re.findall(re.compile(REGEX_ARG), used))
3432
args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL)
@@ -46,8 +44,8 @@ def lint_missing_argument_documentation():
4644

4745

4846
def lint_missing_hidden_wallet_args():
49-
wallet_args = check_output(CMD_GREP_WALLET_ARGS, shell=True).decode('utf8').strip()
50-
wallet_hidden_args = check_output(CMD_GREP_WALLET_HIDDEN_ARGS, shell=True).decode('utf8').strip()
47+
wallet_args = check_output(CMD_GREP_WALLET_ARGS, shell=True, text=True).strip()
48+
wallet_hidden_args = check_output(CMD_GREP_WALLET_HIDDEN_ARGS, shell=True, text=True).strip()
5149

5250
wallet_args = set(re.findall(re.compile(REGEX_DOC), wallet_args))
5351
wallet_hidden_args = set(re.findall(re.compile(r' "([^"=]+)'), wallet_hidden_args))

‎test/lint/lint-files.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def get_git_file_metadata() -> dict[str, FileMeta]:
7676
'''
7777
Return a dictionary mapping the name of all files in the repository to git tree metadata.
7878
'''
79-
files_raw = check_output(CMD_ALL_FILES).decode("utf8").rstrip("\0").split("\0")
79+
files_raw = check_output(CMD_ALL_FILES, text=True).rstrip("\0").split("\0")
8080
files = {}
8181
for file_spec in files_raw:
8282
meta = FileMeta(file_spec)
@@ -169,7 +169,7 @@ def check_shebang_file_permissions(files_meta) -> int:
169169
"""
170170
Checks every file that contains a shebang line to ensure it has an executable permission
171171
"""
172-
filenames = check_output(CMD_SHEBANG_FILES).decode("utf8").strip().split("\n")
172+
filenames = check_output(CMD_SHEBANG_FILES, text=True).strip().split("\n")
173173

174174
# The git grep command we use returns files which contain a shebang on any line within the file
175175
# so we need to filter the list to only files with the shebang on the first line
@@ -198,7 +198,7 @@ def check_shebang_file_permissions(files_meta) -> int:
198198

199199

200200
def main() -> NoReturn:
201-
root_dir = check_output(CMD_TOP_LEVEL).decode("utf8").strip()
201+
root_dir = check_output(CMD_TOP_LEVEL, text=True).strip()
202202
os.chdir(root_dir)
203203

204204
files = get_git_file_metadata()

‎test/lint/lint-include-guards.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def _get_header_file_lst() -> list[str]:
3131
"""
3232
git_cmd_lst = ['git', 'ls-files', '--', '*.h']
3333
header_file_lst = check_output(
34-
git_cmd_lst).decode('utf-8').splitlines()
34+
git_cmd_lst, text=True).splitlines()
3535

3636
header_file_lst = [hf for hf in header_file_lst
3737
if not any(ef in hf for ef

‎test/lint/lint-python-dead-code.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python3
22
#
3-
# Copyright (c) 2022 The Bitcoin Core developers
3+
# Copyright (c) 2022-present The Bitcoin Core developers
44
# Distributed under the MIT software license, see the accompanying
55
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
66

@@ -24,15 +24,15 @@ def check_vulture_install():
2424
def main():
2525
check_vulture_install()
2626

27-
files = check_output(FILES_ARGS).decode("utf-8").splitlines()
27+
files = check_output(FILES_ARGS, text=True).splitlines()
2828
# --min-confidence 100 will only report code that is guaranteed to be unused within the analyzed files.
2929
# Any value below 100 introduces the risk of false positives, which would create an unacceptable maintenance burden.
3030
vulture_args = ['vulture', '--min-confidence=100'] + files
3131

3232
try:
33-
check_output(vulture_args, stderr=STDOUT)
33+
check_output(vulture_args, stderr=STDOUT, text=True)
3434
except CalledProcessError as e:
35-
print(e.output.decode("utf-8"), end="")
35+
print(e.output, end="")
3636
print("Python dead code detection found some issues")
3737
exit(1)
3838

‎test/lint/lint-python.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/usr/bin/env python3
22
#
3-
# Copyright (c) 2022 The Bitcoin Core developers
3+
# Copyright (c) 2022-present The Bitcoin Core developers
44
# Distributed under the MIT software license, see the accompanying
55
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
66

@@ -37,7 +37,7 @@ def check_dependencies():
3737
def main():
3838
check_dependencies()
3939

40-
mypy_files = subprocess.check_output(MYPY_FILES_ARGS).decode("utf-8").splitlines()
40+
mypy_files = subprocess.check_output(MYPY_FILES_ARGS, text=True).splitlines()
4141
mypy_args = ['mypy', '--show-error-codes'] + mypy_files
4242

4343
try:

‎test/lint/lint-shell-locale.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@ def get_shell_files_list():
2727
'*.sh',
2828
]
2929
try:
30-
return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines()
30+
return subprocess.check_output(command, stderr = subprocess.STDOUT, text=True).splitlines()
3131
except subprocess.CalledProcessError as e:
3232
if e.returncode > 1: # return code is 1 when match is empty
33-
print(e.output.decode('utf-8'), end='')
33+
print(e.output, end='')
3434
sys.exit(1)
3535
return []
3636

0 commit comments

Comments
 (0)