...
|
...
|
@@ -103,8 +103,8 @@ class BaseRunner(metaclass=abc.ABCMeta):
|
103
|
103
|
in ``runner.tracebacks``.
|
104
|
104
|
tracebacks : list of tuples
|
105
|
105
|
List of of ``(point, tb)`` for points that failed.
|
106
|
|
- pending_points : dict
|
107
|
|
- A mapping of `~concurrent.futures.Future`\s to points.
|
|
106
|
+ pending_points : list of tuples
|
|
107
|
+ A list of tuples with ``(concurrent.futures.Future, point)``.
|
108
|
108
|
|
109
|
109
|
Methods
|
110
|
110
|
-------
|
...
|
...
|
@@ -132,7 +132,7 @@ class BaseRunner(metaclass=abc.ABCMeta):
|
132
|
132
|
|
133
|
133
|
self._max_tasks = ntasks
|
134
|
134
|
|
135
|
|
- self.pending_points = {}
|
|
135
|
+ self._pending_points = {}
|
136
|
136
|
|
137
|
137
|
# if we instantiate our own executor, then we are also responsible
|
138
|
138
|
# for calling 'shutdown'
|
...
|
...
|
@@ -177,7 +177,7 @@ class BaseRunner(metaclass=abc.ABCMeta):
|
177
|
177
|
pids = [
|
178
|
178
|
pid
|
179
|
179
|
for pid in self._to_retry.keys()
|
180
|
|
- if pid not in self.pending_points.values()
|
|
180
|
+ if pid not in self._pending_points.values()
|
181
|
181
|
][:n]
|
182
|
182
|
loss_improvements = len(pids) * [float("inf")]
|
183
|
183
|
|
...
|
...
|
@@ -215,7 +215,7 @@ class BaseRunner(metaclass=abc.ABCMeta):
|
215
|
215
|
|
216
|
216
|
def _process_futures(self, done_futs):
|
217
|
217
|
for fut in done_futs:
|
218
|
|
- pid = self.pending_points.pop(fut)
|
|
218
|
+ pid = self._pending_points.pop(fut)
|
219
|
219
|
try:
|
220
|
220
|
y = fut.result()
|
221
|
221
|
t = time.time() - fut.start_time # total execution time
|
...
|
...
|
@@ -239,7 +239,7 @@ class BaseRunner(metaclass=abc.ABCMeta):
|
239
|
239
|
# Launch tasks to replace the ones that completed
|
240
|
240
|
# on the last iteration, making sure to fill workers
|
241
|
241
|
# that have started since the last iteration.
|
242
|
|
- n_new_tasks = max(0, self._get_max_tasks() - len(self.pending_points))
|
|
242
|
+ n_new_tasks = max(0, self._get_max_tasks() - len(self._pending_points))
|
243
|
243
|
|
244
|
244
|
if self.do_log:
|
245
|
245
|
self.log.append(("ask", n_new_tasks))
|
...
|
...
|
@@ -251,17 +251,17 @@ class BaseRunner(metaclass=abc.ABCMeta):
|
251
|
251
|
point = self._id_to_point[pid]
|
252
|
252
|
fut = self._submit(point)
|
253
|
253
|
fut.start_time = start_time
|
254
|
|
- self.pending_points[fut] = pid
|
|
254
|
+ self._pending_points[fut] = pid
|
255
|
255
|
|
256
|
256
|
# Collect and results and add them to the learner
|
257
|
|
- futures = list(self.pending_points.keys())
|
|
257
|
+ futures = list(self._pending_points.keys())
|
258
|
258
|
return futures
|
259
|
259
|
|
260
|
260
|
def _remove_unfinished(self):
|
261
|
261
|
# remove points with 'None' values from the learner
|
262
|
262
|
self.learner.remove_unfinished()
|
263
|
263
|
# cancel any outstanding tasks
|
264
|
|
- remaining = list(self.pending_points.keys())
|
|
264
|
+ remaining = list(self._pending_points.keys())
|
265
|
265
|
for fut in remaining:
|
266
|
266
|
fut.cancel()
|
267
|
267
|
return remaining
|
...
|
...
|
@@ -298,11 +298,17 @@ class BaseRunner(metaclass=abc.ABCMeta):
|
298
|
298
|
|
299
|
299
|
@property
|
300
|
300
|
def tracebacks(self):
|
301
|
|
- return [(self._id_to_point[i], tb) for i, tb in self._tracebacks.items()]
|
|
301
|
+ return [(self._id_to_point[pid], tb) for pid, tb in self._tracebacks.items()]
|
302
|
302
|
|
303
|
303
|
@property
|
304
|
304
|
def to_retry(self):
|
305
|
|
- return [(self._id_to_point[i], n) for i, n in self._to_retry.items()]
|
|
305
|
+ return [(self._id_to_point[pid], n) for pid, n in self._to_retry.items()]
|
|
306
|
+
|
|
307
|
+ @property
|
|
308
|
+ def pending_points(self):
|
|
309
|
+ return [
|
|
310
|
+ (fut, self._id_to_point[pid]) for fut, pid in self._pending_points.items()
|
|
311
|
+ ]
|
306
|
312
|
|
307
|
313
|
|
308
|
314
|
class BlockingRunner(BaseRunner):
|
...
|
...
|
@@ -343,14 +349,14 @@ class BlockingRunner(BaseRunner):
|
343
|
349
|
log : list or None
|
344
|
350
|
Record of the method calls made to the learner, in the format
|
345
|
351
|
``(method_name, *args)``.
|
346
|
|
- to_retry : dict
|
347
|
|
- Mapping of ``{point: n_fails, ...}``. When a point has failed
|
|
352
|
+ to_retry : list of tuples
|
|
353
|
+ List of ``(point, n_fails)``. When a point has failed
|
348
|
354
|
``runner.retries`` times it is removed but will be present
|
349
|
355
|
in ``runner.tracebacks``.
|
350
|
|
- tracebacks : dict
|
351
|
|
- A mapping of point to the traceback if that point failed.
|
352
|
|
- pending_points : dict
|
353
|
|
- A mapping of `~concurrent.futures.Future`\to points.
|
|
356
|
+ tracebacks : list of tuples
|
|
357
|
+ List of of ``(point, tb)`` for points that failed.
|
|
358
|
+ pending_points : list of tuples
|
|
359
|
+ A list of tuples with ``(concurrent.futures.Future, point)``.
|
354
|
360
|
|
355
|
361
|
Methods
|
356
|
362
|
-------
|
...
|
...
|
@@ -466,14 +472,14 @@ class AsyncRunner(BaseRunner):
|
466
|
472
|
log : list or None
|
467
|
473
|
Record of the method calls made to the learner, in the format
|
468
|
474
|
``(method_name, *args)``.
|
469
|
|
- to_retry : dict
|
470
|
|
- Mapping of ``{point: n_fails, ...}``. When a point has failed
|
|
475
|
+ to_retry : list of tuples
|
|
476
|
+ List of ``(point, n_fails)``. When a point has failed
|
471
|
477
|
``runner.retries`` times it is removed but will be present
|
472
|
478
|
in ``runner.tracebacks``.
|
473
|
|
- tracebacks : dict
|
474
|
|
- A mapping of point to the traceback if that point failed.
|
475
|
|
- pending_points : dict
|
476
|
|
- A mapping of `~concurrent.futures.Future`\s to points.
|
|
479
|
+ tracebacks : list of tuples
|
|
480
|
+ List of of ``(point, tb)`` for points that failed.
|
|
481
|
+ pending_points : list of tuples
|
|
482
|
+ A list of tuples with ``(concurrent.futures.Future, point)``.
|
477
|
483
|
|
478
|
484
|
Methods
|
479
|
485
|
-------
|