Python源码示例:concurrent.futures.ALL_COMPLETED
示例1
def _close_input_devices(self):
if not hasattr(self, '_opened') or not self._opened:
return
self._opened = False
for event_device in self._event_devices:
asyncio.get_event_loop().remove_reader(event_device.fileno())
event_device.close()
tasks = []
for task in self._tasks:
if not task.done():
task.cancel()
tasks.append(task)
await asyncio.wait(tasks, return_when=futures.ALL_COMPLETED)
self._event_devices.clear()
示例2
def stop(self):
if self.renderer.running:
tasks = []
if self.task is not None and not self.task.done():
self.task.cancel()
tasks.append(self.task)
if self.waiter is not None and not self.waiter.done():
self.waiter.cancel()
tasks.append(self.waiter)
await self.renderer._stop()
if tasks:
await asyncio.wait(tasks, return_when=futures.ALL_COMPLETED)
self.renderer.finish(self._frame)
示例3
def _stop(self):
"""
Stop this AnimationLoop
Shuts down the loop and triggers cleanup tasks.
"""
if not self.running:
return False
self.running = False
for layer in self.layers[::-1]:
await self.remove_layer(layer)
if self._anim_task is not None and not self._anim_task.done():
self._anim_task.cancel()
await asyncio.wait([self._anim_task], return_when=futures.ALL_COMPLETED)
self._logger.info("AnimationLoop stopped")
示例4
def test_all_completed(self):
future1 = self.executor.submit(divmod, 2, 0)
future2 = self.executor.submit(mul, 2, 21)
finished, pending = futures.wait(
[SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2],
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2]), finished)
self.assertEqual(set(), pending)
示例5
def test_timeout(self):
future1 = self.executor.submit(mul, 6, 7)
future2 = self.executor.submit(time.sleep, 6)
finished, pending = futures.wait(
[CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1, future2],
timeout=5,
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1]), finished)
self.assertEqual(set([future2]), pending)
示例6
def create_pool(self, job, parameters, workers=None):
""" Create resources in pool in sub processes.
:param workers:
:type parameters: iterable
:type job: func
"""
executor = ThreadPoolExecutor(
workers) if workers else ThreadPoolExecutor()
try:
# futures = [executor.submit(func, i, kwargs) for i in args]
futures = []
for param_chunk in parameters:
param_chunk['self'] = self
futures.append(executor.submit(job, param_chunk))
concurrent.futures.wait(futures, return_when=ALL_COMPLETED)
responses = {}
for future in futures:
result = future.result()
if result:
responses.update(result)
return responses
finally:
executor.shutdown(wait=True)
示例7
def assemble_python_lambdas(project_path, bundles_dir):
from syndicate.core import CONFIG
project_base_folder = os.path.basename(os.path.normpath(project_path))
project_abs_path = build_path(CONFIG.project_path, project_path)
_LOG.info('Going to process python project by path: {0}'.format(
project_abs_path))
executor = ThreadPoolExecutor(max_workers=5)
futures = []
for root, sub_dirs, files in os.walk(project_abs_path):
for item in files:
if item.endswith(LAMBDA_CONFIG_FILE_NAME):
_LOG.info('Going to build artifact in: {0}'.format(root))
arg = {
'item': item,
'project_base_folder': project_base_folder,
'project_path': project_path,
'root': root,
'target_folder': bundles_dir
}
futures.append(executor.submit(_build_python_artifact, arg))
concurrent.futures.wait(futures, return_when=ALL_COMPLETED)
executor.shutdown()
_LOG.info('Python project was processed successfully')
示例8
def sed_msg(mobile, set_of_times, interval_time, remainder):
init_mobile(mobile, set_of_times)
available_msg = []
while len(available_msg) < remainder:
m = random.choice(all_message_info)
if check_available(m):
available_msg.append(m)
pool = ThreadPoolExecutor(500)
SHARE_Q = queue.Queue(remainder)
while SHARE_Q.qsize() < remainder:
SHARE_Q.put(random.choice(available_msg))
all_task = []
for t in range(SHARE_Q.qsize()):
mg = SHARE_Q.get()
try:
all_task.append(pool.submit(eval(mg), mg, mobile))
except KeyboardInterrupt:
r.delete('%s_%d' % (mobile, set_of_times))
sys.exit(0)
if interval_time:
time.sleep(interval_time)
wait(all_task, return_when=ALL_COMPLETED)
示例9
def test_all_completed(self):
future1 = self.executor.submit(divmod, 2, 0)
future2 = self.executor.submit(mul, 2, 21)
finished, pending = futures.wait(
[SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2],
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2]), finished)
self.assertEqual(set(), pending)
示例10
def test_timeout(self):
future1 = self.executor.submit(mul, 6, 7)
future2 = self.executor.submit(time.sleep, 6)
finished, pending = futures.wait(
[CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1, future2],
timeout=5,
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1]), finished)
self.assertEqual(set([future2]), pending)
示例11
def _create_and_install_waiters(fs, return_when):
if return_when == _AS_COMPLETED:
waiter = _AsCompletedWaiter()
elif return_when == FIRST_COMPLETED:
waiter = _FirstCompletedWaiter()
else:
pending_count = sum(
f._state not in [CANCELLED_AND_NOTIFIED, FINISHED]
for f in fs)
if return_when == FIRST_EXCEPTION:
waiter = _AllCompletedWaiter(pending_count,
stop_on_exception=True)
elif return_when == ALL_COMPLETED:
waiter = _AllCompletedWaiter(pending_count,
stop_on_exception=False)
else:
raise ValueError("Invalid return condition: %r" % return_when)
for f in fs:
f._waiters.append(waiter)
return waiter
示例12
def test_timeout(self):
# Make sure the executor has already started to avoid timeout happening
# before future1 returns
assert self.executor.submit(id_sleep, 42).result() == 42
future1 = self.executor.submit(mul, 6, 7)
future2 = self.executor.submit(self.wait_and_return, 5)
assert future1.result() == 42
finished, pending = futures.wait([CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE, SUCCESSFUL_FUTURE,
future1, future2],
timeout=.1,
return_when=futures.ALL_COMPLETED)
assert set([CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE, future1]) == finished
assert set([future2]) == pending
_executor_mixin._test_event.set()
assert future2.result(timeout=10)
_executor_mixin._test_event.clear()
示例13
def test_all_completed(self):
future1 = self.executor.submit(divmod, 2, 0)
future2 = self.executor.submit(mul, 2, 21)
finished, pending = futures.wait(
[SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2],
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2]), finished)
self.assertEqual(set(), pending)
示例14
def test_timeout(self):
future1 = self.executor.submit(mul, 6, 7)
future2 = self.executor.submit(time.sleep, 6)
finished, pending = futures.wait(
[CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1, future2],
timeout=5,
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1]), finished)
self.assertEqual(set([future2]), pending)
示例15
def test_all_completed(self):
future1 = self.executor.submit(divmod, 2, 0)
future2 = self.executor.submit(mul, 2, 21)
finished, pending = futures.wait(
[SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2],
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([SUCCESSFUL_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
future1,
future2]), finished)
self.assertEqual(set(), pending)
示例16
def test_timeout(self):
future1 = self.executor.submit(mul, 6, 7)
future2 = self.executor.submit(time.sleep, 6)
finished, pending = futures.wait(
[CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1, future2],
timeout=5,
return_when=futures.ALL_COMPLETED)
self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE,
SUCCESSFUL_FUTURE,
future1]), finished)
self.assertEqual(set([future2]), pending)
示例17
def discover(self, timeout=None): # TODO: better name?
"""Discover sentinels and all monitored services within given timeout.
If no sentinels discovered within timeout: TimeoutError is raised.
If some sentinels were discovered but not all — it is ok.
If not all monitored services (masters/slaves) discovered
(or connections established) — it is ok.
TBD: what if some sentinels/services unreachable;
"""
# TODO: check not closed
# TODO: discovery must be done with some customizable timeout.
if timeout is None:
timeout = self.discover_timeout
tasks = []
pools = []
for addr in self._sentinels: # iterate over unordered set
tasks.append(self._connect_sentinel(addr, timeout, pools))
done, pending = await asyncio.wait(tasks,
return_when=ALL_COMPLETED)
assert not pending, ("Expected all tasks to complete", done, pending)
for task in done:
result = task.result()
if isinstance(result, Exception):
continue # FIXME
if not pools:
raise Exception("Could not connect to any sentinel")
pools, self._pools[:] = self._pools[:], pools
# TODO: close current connections
for pool in pools:
pool.close()
await pool.wait_closed()
# TODO: discover peer sentinels
for pool in self._pools:
await pool.execute_pubsub(
b'psubscribe', self._monitor.pattern('*'))
示例18
def wait_tasks(futures, timeout=None, return_when=ALL_COMPLETED, raise_exceptions=True):
running_futures = [fut for fut in futures if not fut.done()]
done, _ = wait(running_futures, timeout=timeout, return_when=return_when)
if raise_exceptions:
[f.result() for f in done if not f.cancelled() and f.exception() is not None] # raises the exception
示例19
def wait_tasks_or_abort(futures, timeout=60, kill_switch_ev=None):
try:
LazySingletonTasksCoordinator.wait_tasks(futures, return_when=FIRST_EXCEPTION, raise_exceptions=True)
except Exception as e:
if kill_switch_ev is not None:
# Used when we want to keep both raise the exception and wait for all tasks to finish
kill_switch_ev.set()
LazySingletonTasksCoordinator.wait_tasks(futures, return_when=ALL_COMPLETED,
raise_exceptions=False, timeout=timeout)
raise e
示例20
def await_termination(self, timeout=None):
with type(self)._POOL_LOCK:
if not self.is_shutdown:
raise AsyncArcticException("The workers pool has not been shutdown, please call shutdown() first.")
LazySingletonTasksCoordinator.wait_tasks(
[v[0] for v in itervalues(self.alive_tasks)],
timeout=timeout, return_when=ALL_COMPLETED, raise_exceptions=False)
with type(self)._POOL_LOCK:
self.alive_tasks = {}
示例21
def _run(self):
dm = UChromaDeviceManager()
atexit.register(UChromaServer.exit, self._loop)
dbus = DeviceManagerAPI(dm, self._logger)
power = PowerMonitor()
for sig in (signal.SIGINT, signal.SIGTERM):
self._loop.add_signal_handler(sig, self._shutdown_callback)
try:
dbus.run()
power.start()
ensure_future(dm.monitor_start(), loop=self._loop)
self._loop.run_forever()
except KeyboardInterrupt:
pass
finally:
for sig in (signal.SIGTERM, signal.SIGINT):
self._loop.remove_signal_handler(sig)
power.stop()
self._loop.run_until_complete(asyncio.wait( \
[dm.close_devices(), dm.monitor_stop()],
return_when=futures.ALL_COMPLETED))
示例22
def exit(loop):
try:
loop.run_until_complete(asyncio.wait( \
list(asyncio.Task.all_tasks()),
return_when=futures.ALL_COMPLETED))
loop.close()
except KeyboardInterrupt:
pass
示例23
def test_pending_calls_race(self):
# Issue #14406: multi-threaded race condition when waiting on all
# futures.
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)
示例24
def _apply_dynamic_changes(resources, output):
from syndicate.core import PROCESSOR_FACADE
pool = ThreadPoolExecutor(max_workers=5)
futures = []
for name, meta in resources.items():
resource_type = meta['resource_type']
apply_changes = meta.get('apply_changes')
if apply_changes:
for apply_item in apply_changes:
change_type = apply_item['apply_type']
dependency_name = apply_item['dependency_name']
res_config = resources.get(dependency_name)
if not res_config:
_LOG.debug('Dependency resource {0} is not found, '
'skipping the apply'.format(dependency_name))
else:
dependency_type = res_config['resource_type']
func = PROCESSOR_FACADE.resource_identifier() \
.get(resource_type)
if func:
resource_output = __find_output_by_resource_name(
output, name)
identifier = func(name, resource_output)
apply_func = PROCESSOR_FACADE.mapping_applier() \
.get(change_type)
if apply_func:
alias = '#{' + name + '}'
f = pool.submit(apply_func, alias, identifier,
apply_item)
futures.append(f)
else:
_LOG.warn('Dynamic apply is not defined '
'for {0} type'.format(change_type))
else:
_LOG.warn('Resource identifier is not defined '
'for {0} type'.format(dependency_type))
_LOG.info('Dynamic changes were applied to {0}'.format(name))
concurrent.futures.wait(futures, timeout=None, return_when=ALL_COMPLETED)
示例25
def test_pending_calls_race(self):
# Issue #14406: multi-threaded race condition when waiting on all
# futures.
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)
示例26
def wait_for_all(fs, timeout=None):
"""Wait for all of the futures to complete.
Works correctly with both green and non-green futures (but not both
together, since this can't be guaranteed to avoid dead-lock due to how
the waiting implementations are different when green threads are being
used).
Returns pair (done futures, not done futures).
"""
return _wait_for(fs, futures.ALL_COMPLETED, _wait_for_all_green,
'wait_for_all', timeout=timeout)
示例27
def test_all_completed(self):
future1 = self.executor.submit(divmod, 2, 0)
future2 = self.executor.submit(mul, 2, 21)
finished, pending = futures.wait([SUCCESSFUL_FUTURE, EXCEPTION_FUTURE,
CANCELLED_AND_NOTIFIED_FUTURE,
future1, future2],
return_when=futures.ALL_COMPLETED)
assert set([SUCCESSFUL_FUTURE, CANCELLED_AND_NOTIFIED_FUTURE,
EXCEPTION_FUTURE, future1, future2]) == finished
assert set() == pending
示例28
def test_pending_calls_race(self):
# Issue #14406: multi-threaded race condition when waiting on all
# futures.
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)
示例29
def wait(fs, timeout=None, return_when=ALL_COMPLETED):
raise NotImplementedError()
示例30
def test_pending_calls_race(self):
# Issue #14406: multi-threaded race condition when waiting on all
# futures.
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)