add new SEARCH_BACKEND_TIMEOUT config option defaulted to 90sec

pirate · pirate · commit 32764347ce2e · 2021-04-09T00:27:12.000-04:00
diff --git a/archivebox/config.py b/archivebox/config.py
@@ -157,6 +157,7 @@
         # SONIC
         'SONIC_COLLECTION':         {'type': str,   'default': 'archivebox'},
         'SONIC_BUCKET':             {'type': str,   'default': 'snapshots'},
+        'SEARCH_BACKEND_TIMEOUT':   {'type': int,   'default': 90},
     },
 
     'DEPENDENCY_CONFIG': {
diff --git a/archivebox/search/backends/ripgrep.py b/archivebox/search/backends/ripgrep.py
@@ -2,7 +2,7 @@
 from subprocess import run, PIPE
 from typing import List, Generator
 
-from archivebox.config import ARCHIVE_DIR, RIPGREP_VERSION
+from archivebox.config import ARCHIVE_DIR, RIPGREP_VERSION, SEARCH_BACKEND_TIMEOUT
 from archivebox.util import enforce_types
 
 RG_IGNORE_EXTENSIONS = ('css','js','orig','svg')
@@ -32,7 +32,7 @@ def search(text: str) -> List[str]:
     from core.models import Snapshot
 
     rg_cmd = ['rg', RG_ADD_TYPE, RG_IGNORE_ARGUMENTS, RG_DEFAULT_ARGUMENTS, RG_REGEX_ARGUMENT, text, str(ARCHIVE_DIR)]
-    rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=60)
+    rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=SEARCH_BACKEND_TIMEOUT)
     file_paths = [p.decode() for p in rg.stdout.splitlines()]
     timestamps = set()
     for path in file_paths:
