Skip to content

Commit e478131

Browse files
Makman2Makman2
committed
core: Add DependencyBear
Allows to submit tasks on a per-dependency-result-basis. Closes #4425
1 parent abb9fe6 commit e478131

2 files changed

Lines changed: 262 additions & 0 deletions

File tree

‎coalib/core/DependencyBear.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
from coalib.core.Bear import Bear
2+
from coalib.settings.FunctionMetadata import FunctionMetadata
3+
4+
5+
class DependencyBear(Bear):
6+
"""
7+
This bear base class parallelizes tasks for each dependency result.
8+
9+
You can specify dependency bears with the ``BEAR_DEPS`` field.
10+
"""
11+
12+
def __init__(self, section, file_dict):
13+
"""
14+
:param section:
15+
The section object where bear settings are contained. A section
16+
passed here is considered to be immutable.
17+
:param file_dict:
18+
A dictionary containing filenames to process as keys and their
19+
contents (line-split with trailing return characters) as values.
20+
"""
21+
Bear.__init__(self, section, file_dict)
22+
23+
self._kwargs = self.get_metadata().create_params_from_section(section)
24+
25+
def execute_task(self, args, kwargs):
26+
# To optimize performance and memory usage, we use kwargs from this
27+
# class instance instead of passing the same settings thousand times in
28+
# the tasks themselves.
29+
return Bear.execute_task(self, args, self._kwargs)
30+
31+
@classmethod
32+
def get_metadata(cls):
33+
"""
34+
:return:
35+
Metadata for the ``analyze`` function extracted from its signature.
36+
Excludes parameters ``self``, ``filename`` and ``file``.
37+
"""
38+
return FunctionMetadata.from_function(
39+
cls.analyze,
40+
omit={'self', 'dependency_bear', 'dependency_result'})
41+
42+
def generate_tasks(self):
43+
return (((bear, dependency_result), {})
44+
for bear, dependency_results in self.dependency_results.items()
45+
for dependency_result in dependency_results)

‎tests/core/DependencyBearTest.py‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
from concurrent.futures import ThreadPoolExecutor
2+
3+
from coalib.core.DependencyBear import DependencyBear
4+
from coalib.core.FileBear import FileBear
5+
from coalib.core.ProjectBear import ProjectBear
6+
from coalib.settings.Section import Section
7+
8+
from tests.core.CoreTestBase import CoreTestBase
9+
10+
11+
class TestProjectBear(ProjectBear):
12+
13+
def analyze(self, files):
14+
yield ', '.join('{}({})'.format(filename, len(files[filename]))
15+
for filename in sorted(files))
16+
17+
18+
class TestFileBear(FileBear):
19+
20+
def analyze(self, filename, file):
21+
yield '{}:{}'.format(filename, len(file))
22+
23+
24+
class TestBearDependentOnProjectBear(DependencyBear):
25+
BEAR_DEPS = {TestProjectBear}
26+
27+
def analyze(self, dependency_bear, dependency_result):
28+
yield '{} - {}'.format(dependency_bear.name, dependency_result)
29+
30+
31+
class TestBearDependentOnFileBear(DependencyBear):
32+
BEAR_DEPS = {TestFileBear}
33+
34+
def analyze(self, dependency_bear, dependency_result):
35+
yield '{} - {}'.format(dependency_bear.name, dependency_result)
36+
37+
38+
class TestBearDependentOnMultipleBears(DependencyBear):
39+
BEAR_DEPS = {TestFileBear, TestProjectBear}
40+
41+
def analyze(self, dependency_bear, dependency_result, a_number=100):
42+
yield '{} ({}) - {}'.format(
43+
dependency_bear.name, a_number, dependency_result)
44+
45+
46+
class DependencyBearTest(CoreTestBase):
47+
48+
def assertResultsEqual(self, bear_type, expected,
49+
section=None, file_dict=None):
50+
"""
51+
Asserts whether the expected results do match the output of the bear.
52+
53+
Asserts for the results out-of-order.
54+
55+
:param bear_type:
56+
The bear class to check.
57+
:param expected:
58+
A sequence of expected results.
59+
:param section:
60+
A section for the bear to use. By default uses a new section with
61+
name ``test-section``.
62+
:param file_dict:
63+
A file-dictionary for the bear to use. By default uses an empty
64+
dictionary.
65+
"""
66+
if section is None:
67+
section = Section('test-section')
68+
if file_dict is None:
69+
file_dict = {}
70+
71+
uut = bear_type(section, file_dict)
72+
73+
results = self.execute_run({uut})
74+
75+
self.assertEqual(sorted(expected), sorted(results))
76+
77+
def test_projectbear_dependency(self):
78+
# Dependency results are also catched in the result callback, thus they
79+
# are included in the final result list.
80+
self.assertResultsEqual(
81+
TestBearDependentOnProjectBear,
82+
file_dict={},
83+
expected=['',
84+
'TestProjectBear - '])
85+
self.assertResultsEqual(
86+
TestBearDependentOnProjectBear,
87+
file_dict={'fileX': []},
88+
expected=['fileX(0)',
89+
'TestProjectBear - fileX(0)'])
90+
self.assertResultsEqual(
91+
TestBearDependentOnProjectBear,
92+
file_dict={'fileX': [], 'fileY': ['hello']},
93+
expected=['fileX(0), fileY(1)',
94+
'TestProjectBear - fileX(0), fileY(1)'])
95+
self.assertResultsEqual(
96+
TestBearDependentOnProjectBear,
97+
file_dict={'fileX': [], 'fileY': ['hello'], 'fileZ': ['x\n', 'y']},
98+
expected=['fileX(0), fileY(1), fileZ(2)',
99+
'TestProjectBear - fileX(0), fileY(1), fileZ(2)'])
100+
101+
def test_filebear_dependency(self):
102+
# Dependency results are also catched in the result callback, thus they
103+
# are included in the final result list.
104+
self.assertResultsEqual(
105+
TestBearDependentOnFileBear,
106+
file_dict={},
107+
expected=[])
108+
self.assertResultsEqual(
109+
TestBearDependentOnFileBear,
110+
file_dict={'fileX': []},
111+
expected=['fileX:0',
112+
'TestFileBear - fileX:0'])
113+
self.assertResultsEqual(
114+
TestBearDependentOnFileBear,
115+
file_dict={'fileX': [], 'fileY': ['hello']},
116+
expected=['fileX:0',
117+
'fileY:1',
118+
'TestFileBear - fileX:0',
119+
'TestFileBear - fileY:1'])
120+
self.assertResultsEqual(
121+
TestBearDependentOnFileBear,
122+
file_dict={'fileX': [], 'fileY': ['hello'], 'fileZ': ['x\n', 'y']},
123+
expected=['fileX:0',
124+
'fileY:1',
125+
'fileZ:2',
126+
'TestFileBear - fileX:0',
127+
'TestFileBear - fileY:1',
128+
'TestFileBear - fileZ:2'])
129+
130+
def test_multiple_bears_dependencies(self):
131+
# Dependency results are also catched in the result callback, thus they
132+
# are included in the final result list.
133+
self.assertResultsEqual(
134+
TestBearDependentOnMultipleBears,
135+
file_dict={},
136+
expected=['',
137+
'TestProjectBear (100) - '])
138+
self.assertResultsEqual(
139+
TestBearDependentOnMultipleBears,
140+
file_dict={'fileX': []},
141+
expected=['fileX(0)',
142+
'TestProjectBear (100) - fileX(0)',
143+
'fileX:0',
144+
'TestFileBear (100) - fileX:0'])
145+
self.assertResultsEqual(
146+
TestBearDependentOnMultipleBears,
147+
file_dict={'fileX': [], 'fileY': ['hello']},
148+
expected=['fileX(0), fileY(1)',
149+
'TestProjectBear (100) - fileX(0), fileY(1)',
150+
'fileX:0',
151+
'fileY:1',
152+
'TestFileBear (100) - fileX:0',
153+
'TestFileBear (100) - fileY:1'])
154+
self.assertResultsEqual(
155+
TestBearDependentOnMultipleBears,
156+
file_dict={'fileX': [], 'fileY': ['hello'], 'fileZ': ['x\n', 'y']},
157+
expected=['fileX(0), fileY(1), fileZ(2)',
158+
'TestProjectBear (100) - fileX(0), fileY(1), fileZ(2)',
159+
'fileX:0',
160+
'fileY:1',
161+
'fileZ:2',
162+
'TestFileBear (100) - fileX:0',
163+
'TestFileBear (100) - fileY:1',
164+
'TestFileBear (100) - fileZ:2'])
165+
166+
def test_multiple_bears_dependencies_with_parameter(self):
167+
section = Section('test-section')
168+
section['a_number'] = '500'
169+
170+
# Dependency results are also catched in the result callback, thus they
171+
# are included in the final result list.
172+
self.assertResultsEqual(
173+
TestBearDependentOnMultipleBears,
174+
section=section,
175+
file_dict={},
176+
expected=['',
177+
'TestProjectBear (500) - '])
178+
self.assertResultsEqual(
179+
TestBearDependentOnMultipleBears,
180+
section=section,
181+
file_dict={'fileX': []},
182+
expected=['fileX(0)',
183+
'TestProjectBear (500) - fileX(0)',
184+
'fileX:0',
185+
'TestFileBear (500) - fileX:0'])
186+
self.assertResultsEqual(
187+
TestBearDependentOnMultipleBears,
188+
section=section,
189+
file_dict={'fileX': [], 'fileY': ['hello']},
190+
expected=['fileX(0), fileY(1)',
191+
'TestProjectBear (500) - fileX(0), fileY(1)',
192+
'fileX:0',
193+
'fileY:1',
194+
'TestFileBear (500) - fileX:0',
195+
'TestFileBear (500) - fileY:1'])
196+
self.assertResultsEqual(
197+
TestBearDependentOnMultipleBears,
198+
section=section,
199+
file_dict={'fileX': [], 'fileY': ['hello'], 'fileZ': ['x\n', 'y']},
200+
expected=['fileX(0), fileY(1), fileZ(2)',
201+
'TestProjectBear (500) - fileX(0), fileY(1), fileZ(2)',
202+
'fileX:0',
203+
'fileY:1',
204+
'fileZ:2',
205+
'TestFileBear (500) - fileX:0',
206+
'TestFileBear (500) - fileY:1',
207+
'TestFileBear (500) - fileZ:2'])
208+
209+
210+
# Execute the same tests from DependencyBearTest, but use a ThreadPoolExecutor
211+
# instead. It shall also seamlessly work with Python threads. Also there are
212+
# coverage issues on Windows with ProcessPoolExecutor as coverage data isn't
213+
# passed properly back from the pool processes.
214+
class DependencyBearOnThreadPoolExecutorTest(DependencyBearTest):
215+
def setUp(self):
216+
super().setUp()
217+
self.executor = ThreadPoolExecutor, tuple(), dict(max_workers=8)

0 commit comments

Comments
 (0)