Python源码示例:asyncio.FIRST_COMPLETED

示例1
def wait_for_first_response(tasks, converters):
    """given a list of unawaited tasks and non-coro result parsers to be called on the results,
    this function returns the 1st result that is returned and converted

    if it is possible for 2 tasks to complete at the same time,
    only the 1st result deteremined by asyncio.wait will be returned

    returns None if none successfully complete
    returns 1st error raised if any occur (probably)
    """
    primed = [wait_for_result(t, c) for t, c in zip(tasks, converters)]
    done, pending = await asyncio.wait(primed, return_when=asyncio.FIRST_COMPLETED)
    for p in pending:
        p.cancel()

    try:
        return done.pop().result()
    except NotImplementedError as e:
        raise e
    except:
        return None 
示例2
def _handler(self, websocket: websockets.WebSocketCommonProtocol, *unused_args):
        """Setup the consumer and producer response handlers with asyncio.

        Args:
            websocket: the websocket connection to the client

        """

        asyncio.set_event_loop(self.loop)
        consumer_task = asyncio.ensure_future(self._consumer_handler(websocket))
        producer_task = asyncio.ensure_future(self._producer_handler(websocket))

        done, pending = await asyncio.wait(
            [consumer_task, producer_task], return_when=asyncio.FIRST_COMPLETED
        )

        for task in pending:
            task.cancel() 
示例3
def stop(self):
        """Flush all pending data and close all connections to kafka cluster"""
        if self._closed:
            return
        self._closed = True

        # If the sender task is down there is no way for accumulator to flush
        if self._sender is not None and self._sender.sender_task is not None:
            await asyncio.wait([
                self._message_accumulator.close(),
                self._sender.sender_task],
                return_when=asyncio.FIRST_COMPLETED,
                loop=self._loop)

            await self._sender.close()

        await self.client.close()
        log.debug("The Kafka producer has closed.") 
示例4
def _push_error_to_user(self, exc):
        """ Most critical errors are not something we can continue execution
        without user action. Well right now we just drop the Consumer, but
        java client would certainly be ok if we just poll another time, maybe
        it will need to rejoin, but not fail with GroupAuthorizationFailedError
        till the end of days...
        XXX: Research if we can't have the same error several times. For
             example if user gets GroupAuthorizationFailedError and adds
             permission for the group, would Consumer work right away or would
             still raise exception a few times?
        """
        exc = copy.copy(exc)
        self._subscription.abort_waiters(exc)
        self._pending_exception = exc
        self._error_consumed_fut = create_future(loop=self._loop)
        return asyncio.wait(
            [self._error_consumed_fut, self._closing],
            return_when=asyncio.FIRST_COMPLETED,
            loop=self._loop
        ) 
示例5
def connect(self, timeout: Optional[float] = None) -> None:
        """Establish a connection with the WebSocket server.

        Returns:
            `(True, <chosen-subprotocol>)` if connection accepted.
            `(False, None)` if connection rejected.

        """
        connected = asyncio.Event()
        self._message_processor = asyncio.ensure_future(
            self._process_messages(connected, timeout or self.TIMEOUT)
        )
        await asyncio.wait(
            [connected.wait(), self._message_processor],
            return_when=asyncio.FIRST_COMPLETED,
        )
        if self._message_processor.done():
            # Make sure to raise an exception from the task.
            self._message_processor.result()
            raise RuntimeError(f"Failed to connect to the server: {self._url}!") 
示例6
def onConnect (self, **kwargs):
        self.logger.info ('connect', nick=self.nick, uuid='01f7b138-ea53-4609-88e9-61f3eca3e7e7')

        self.send('NICK', nick=self.nick)
        self.send('USER', user=self.nick, realname='https://github.com/PromyLOPh/crocoite')

        # Don't try to join channels until the server has
        # sent the MOTD, or signaled that there's no MOTD.
        done, pending = await asyncio.wait(
            [self.wait('RPL_ENDOFMOTD'), self.wait('ERR_NOMOTD')],
            loop=self.loop, return_when=asyncio.FIRST_COMPLETED)

        # Cancel whichever waiter's event didn't come in.
        for future in pending:
            future.cancel()

        for c in self.channels:
            self.logger.info ('join', channel=c, uuid='367063a5-9069-4025-907c-65ba88af8593')
            self.send ('JOIN', channel=c)
            # no need for NAMES here, server sends this automatically 
示例7
def poll(self, events, timeout=None,
                   return_when=asyncio.FIRST_COMPLETED):
        """Poll for any of a set of event types to be received for this session.
        """
        awaitables = {}
        for name in events:
            awaitables[self.recv(name)] = name
        done, pending = await asyncio.wait(
            awaitables, timeout=timeout, return_when=return_when)

        if done:
            ev_dicts = []
            for fut in done:
                awaitables.pop(fut)
                ev_dicts.append(fut.result())
            return ev_dicts, awaitables.values()
        else:
            raise asyncio.TimeoutError(
                "None of {} was received in {} seconds"
                .format(events, timeout))

    # call control / 'mod_commands' methods
    # TODO: dynamically add @decorated functions to this class
    # and wrap them using functools.update_wrapper ...? 
示例8
def cancellable_wait(self, *awaitables: Awaitable[_R], timeout: float = None) -> _R:
        futures = [asyncio.ensure_future(a, loop=self.loop) for a in awaitables + (self.wait(),)]
        try:
            done, pending = await asyncio.wait(
                futures,
                timeout=timeout,
                return_when=asyncio.FIRST_COMPLETED,
                loop=self.loop,
            )
        except CancelledError:
            for future in futures:
                future.cancel()
            raise
        for task in pending:
            task.cancel()
        await asyncio.wait(pending, return_when=asyncio.ALL_COMPLETED, loop=self.loop,)
        if not done:
            raise TimeoutError()
        if self.triggered_token is not None:
            for task in done:
                task.exception()
            raise OperationCancelled(f'Cancellation requested by {self.triggered_token} token')
        return done.pop().result() 
示例9
def ws_handler(self, request):

        ws = web.WebSocketResponse()
        await ws.prepare(request)

        if self.ws is not None:
            await self.ws.close()

        self.ws = ws

        _, unfinished = await asyncio.wait(
            [
                self._websocket_receive(ws),
                self._websocket_send(ws)
            ],
            return_when=asyncio.FIRST_COMPLETED
        )
        for task in unfinished:
            task.cancel()

        return ws 
示例10
def waiter(client):
    async def wait_for(*events, return_when=asyncio.FIRST_COMPLETED):
        if not events:
            return
        done, pending = await asyncio.wait(
            [client.wait(event) for event in events],
            loop=client.loop,
            return_when=return_when)

        # Get the result(s) of the completed task(s).
        ret = [future.result() for future in done]

        # Cancel any events that didn't come in.
        for future in pending:
            future.cancel()

        # Return list of completed event names.
        return ret
    return wait_for


# taken from :ref:`Patterns` 
示例11
def stop(self):
        """Flush all pending data and close all connections to kafka cluster"""
        if self._closed:
            return
        self._closed = True

        # If the sender task is down there is no way for accumulator to flush
        if self._sender is not None and self._sender.sender_task is not None:
            await asyncio.wait([
                self._message_accumulator.close(),
                self._sender.sender_task],
                return_when=asyncio.FIRST_COMPLETED,
                loop=self._loop)

            await self._sender.close()

        await self.client.close()
        log.debug("The Kafka producer has closed.") 
示例12
def _push_error_to_user(self, exc):
        """ Most critical errors are not something we can continue execution
        without user action. Well right now we just drop the Consumer, but
        java client would certainly be ok if we just poll another time, maybe
        it will need to rejoin, but not fail with GroupAuthorizationFailedError
        till the end of days...
        XXX: Research if we can't have the same error several times. For
             example if user gets GroupAuthorizationFailedError and adds
             permission for the group, would Consumer work right away or would
             still raise exception a few times?
        """
        exc = copy.copy(exc)
        self._subscription.abort_waiters(exc)
        self._pending_exception = exc
        self._error_consumed_fut = create_future(loop=self._loop)
        return asyncio.wait(
            [self._error_consumed_fut, self._closing],
            return_when=asyncio.FIRST_COMPLETED,
            loop=self._loop
        ) 
示例13
def daemon():
    try_init_cgroup()

    async with VJ4Session(config['server_url']) as session:
        while True:
            try:
                await session.login_if_needed(config['uname'], config['password'])
                done, pending = await wait([do_judge(session), do_noop(session)],
                                           return_when=FIRST_COMPLETED)
                for task in pending:
                    task.cancel()
                await gather(*done)
            except Exception as e:
                logger.exception(e)
            logger.info('Retrying after %d seconds', RETRY_DELAY_SEC)
            await sleep(RETRY_DELAY_SEC) 
示例14
def _run(self):
        first_completed = concurrent.FIRST_COMPLETED

        if self._get_max_tasks() < 1:
            raise RuntimeError("Executor has no workers")

        try:
            while not self.goal(self.learner):
                futures = self._get_futures()
                done, _ = concurrent.wait(futures, return_when=first_completed)
                self._process_futures(done)
        finally:
            remaining = self._remove_unfinished()
            if remaining:
                concurrent.wait(remaining)
            self._cleanup() 
示例15
def _run(self):
        first_completed = asyncio.FIRST_COMPLETED

        if self._get_max_tasks() < 1:
            raise RuntimeError("Executor has no workers")

        try:
            while not self.goal(self.learner):
                futures = self._get_futures()
                done, _ = await asyncio.wait(
                    futures, return_when=first_completed, loop=self.ioloop
                )
                self._process_futures(done)
        finally:
            remaining = self._remove_unfinished()
            if remaining:
                await asyncio.wait(remaining)
            self._cleanup() 
示例16
def mqtt_connected(func):
    """
        MQTTClient coroutines decorator which will wait until connection before calling the decorated method.
        :param func: coroutine to be called once connected
        :return: coroutine result
    """
    @asyncio.coroutine
    @wraps(func)
    def wrapper(self, *args, **kwargs):
        if not self._connected_state.is_set():
            base_logger.warning("Client not connected, waiting for it")
            _, pending = yield from asyncio.wait([self._connected_state.wait(), self._no_more_connections.wait()], return_when=asyncio.FIRST_COMPLETED)
            for t in pending:
                t.cancel()
            if self._no_more_connections.is_set():
                raise ClientException("Will not reconnect")
        return (yield from func(self, *args, **kwargs))
    return wrapper 
示例17
def __call__(self, receive: Callable, send: Callable) -> None:
        request = self._create_request_from_scope(send)
        receiver_task = asyncio.ensure_future(self.handle_messages(request, receive))
        handler_task = asyncio.ensure_future(self.handle_request(request, send))
        done, pending = await asyncio.wait(
            [handler_task, receiver_task], return_when=asyncio.FIRST_COMPLETED
        )
        await _cancel_tasks(pending)
        _raise_exceptions(done) 
示例18
def __call__(self, receive: Callable, send: Callable) -> None:
        websocket = self._create_websocket_from_scope(send)
        receiver_task = asyncio.ensure_future(self.handle_messages(receive))
        handler_task = asyncio.ensure_future(self.handle_websocket(websocket, send))
        done, pending = await asyncio.wait(
            [handler_task, receiver_task], return_when=asyncio.FIRST_COMPLETED
        )
        await _cancel_tasks(pending)
        _raise_exceptions(done) 
示例19
def process(self, queue, workflow):
        try:
            while queue.__futures__:
                done, _ = await wait(queue.__futures__,
                                     return_when=FIRST_COMPLETED)
                queue.progress(done)
            return workflow.result()
        except CancelledError:
            for task in queue.__futures__:
                task.cancel()
            await gather(*queue.__futures__)
            raise 
示例20
def _connect(self):
        await self.launch_shards()

        while True:
            pollers = [shard.get_future() for shard in self.shards.values()]
            done, _ = await asyncio.wait(pollers, return_when=asyncio.FIRST_COMPLETED)
            for f in done:
                # we wanna re-raise to the main Client.connect handler if applicable
                f.result() 
示例21
def process(self):

        try:

            if await self.connect():

                self._listener_task = asyncio.ensure_future(self._ws.receive())
                self._sender_task = asyncio.ensure_future(self._outgoing_queue.get())

                main_tasks = [self._listener_task, self._sender_task]

                done, pending = await asyncio.wait(main_tasks, return_when=asyncio.FIRST_COMPLETED)

                if self._listener_task in done:
                    self.logger.debug('[transport] receiving :  {}'.format(self._listener_task.result()))
                    await self._incoming_queue.put(self._listener_task.result())
                else:
                    # self.logger.debug("listener task pended")
                    self._listener_task.cancel()

                if self._sender_task in done:
                    message = self._sender_task.result()
                    self._ws.send_str(str(message))
                    self.logger.debug('[transport] sending :  {}'.format(message))
                elif self._sender_task:
                    # self.logger.debug("sender tast pended")
                    self._sender_task.cancel()

        except Exception as ex:
            self.logger.error(ex, extra={"tag": "err"})
            traceback.print_exc() 
示例22
def test_wait(task):
    done, pending = await asyncio.wait([task], return_when=asyncio.FIRST_COMPLETED)
    for d in done:
        d.exception()
    print("test_wait done")


# Test group 1
# task = asyncio.get_event_loop().create_task(start())
# asyncio.get_event_loop().create_task(test_cancel(task))
# asyncio.get_event_loop().create_task(test_cancel(task))

# Test group 2 
示例23
def launch(self):
        event_loop = asyncio.get_event_loop()

        # TODO remove this line once we stop supporting Python 3.5
        asyncio.set_event_loop(event_loop)

        try:
            self._websocket = event_loop.run_until_complete(
                websockets.connect(self._uri))
        except ConnectionRefusedError:
            logger.error('Could not connect to server')
            return

        consumer_task = asyncio.ensure_future(self._consumer_handler())
        tasks = [
            asyncio.ensure_future(self._producer_handler(
                producer_wrapper.producer, producer_wrapper.source_name))
            for producer_wrapper in self.producer_wrappers
        ]
        tasks.append(consumer_task)

        _, pending = event_loop.run_until_complete(asyncio.wait(
                tasks, return_when=asyncio.FIRST_COMPLETED))
        for task in pending:
            task.cancel()
        logger.info('Disconnected From Server') 
示例24
def connect(self):
        """Connect Slack RTM."""
        logger = logging.getLogger(f'{__name__}.Bot.connect')
        sleep = 0

        while True:
            try:
                rtm = await self.call('rtm.start')
            except Exception as e:
                logger.exception(e)
                await asyncio.sleep((sleep + 1) * 10)
                sleep += 1
                continue

            if not rtm.body['ok']:
                await asyncio.sleep((sleep + 1) * 10)
                sleep += 1
                continue
            else:
                sleep = 0

            await self.queue.put(create_event('chatterbox_system_start', {}))
            while not self.is_ready:
                await asyncio.sleep(0.01)
            try:
                async with aiohttp.ClientSession() as session:
                    async with session.ws_connect(rtm.body['url']) as ws:
                        await asyncio.wait(
                            (self.ping(ws), self.receive(ws),),
                            return_when=asyncio.FIRST_COMPLETED,
                        )

                raise BotReconnect()
            except BotReconnect:
                logger.info('BotReconnect raised. I will reconnect to rtm.')
                continue
            except:  # noqa
                logger.exception('Unexpected Exception raised')
                continue 
示例25
def process(self):
        # start websocket connection
        ws = self.ws = web.WebSocketResponse()
        await ws.prepare(self.request)

        # session was interrupted
        if self.session.interrupted:
            await self.ws.send_str(close_frame(1002, "Connection interrupted"))

        elif self.session.state == STATE_CLOSED:
            await self.ws.send_str(close_frame(3000, "Go away!"))

        else:
            try:
                await self.manager.acquire(self.session)
            except Exception:  # should use specific exception
                await self.ws.send_str(close_frame(3000, "Go away!"))
                await ws.close()
                return ws
            server = ensure_future(self.server(ws, self.session))
            client = ensure_future(self.client(ws, self.session))
            try:
                await asyncio.wait(
                    (server, client), return_when=asyncio.FIRST_COMPLETED
                )
            except asyncio.CancelledError:
                raise
            except Exception as exc:
                await self.session._remote_close(exc)
            finally:
                await self.manager.release(self.session)
                if not server.done():
                    server.cancel()
                if not client.done():
                    client.cancel()

        return ws 
示例26
def process(self):
        # start websocket connection
        ws = self.ws = web.WebSocketResponse(autoping=False)
        await ws.prepare(self.request)

        try:
            await self.manager.acquire(self.session)
        except Exception:  # should use specific exception
            await ws.close(message="Go away!")
            return ws

        server = ensure_future(self.server(ws, self.session))
        client = ensure_future(self.client(ws, self.session))
        try:
            await asyncio.wait((server, client), return_when=asyncio.FIRST_COMPLETED)
        except asyncio.CancelledError:
            raise
        except Exception as exc:
            await self.session._remote_close(exc)
        finally:
            await self.manager.release(self.session)
            if not server.done():
                server.cancel()
            if not client.done():
                client.cancel()

        return ws 
示例27
def _reset_committed_routine(self):
        """ Group coordinator will reset committed points to UNKNOWN_OFFSET
        if no commit is found for group. In the NoGroup mode we need to force
        it after each assignment
        """
        event_waiter = None
        try:
            while True:
                if self._subscription.subscription is None:
                    await self._subscription.wait_for_subscription()
                    continue

                assignment = self._subscription.subscription.assignment
                if assignment is None:
                    await self._subscription.wait_for_assignment()
                    continue

                commit_refresh_needed = assignment.commit_refresh_needed
                commit_refresh_needed.clear()

                for tp in assignment.requesting_committed():
                    tp_state = assignment.state_value(tp)
                    tp_state.update_committed(
                        OffsetAndMetadata(UNKNOWN_OFFSET, ""))

                event_waiter = ensure_future(
                    commit_refresh_needed.wait(), loop=self._loop)

                await asyncio.wait(
                    [assignment.unassign_future, event_waiter],
                    return_when=asyncio.FIRST_COMPLETED,
                    loop=self._loop)

        except asyncio.CancelledError:
            pass

        # Just to make sure we properly close started tasks we cancel
        # event.wait() task
        if event_waiter is not None and not event_waiter.done():
            event_waiter.cancel()
            event_waiter = None 
示例28
def _commit_refresh_routine(self, assignment):
        """ Task that will do a commit cache refresh if someone is waiting for
        it.
        """
        retry_backoff_ms = self._retry_backoff_ms / 1000
        commit_refresh_needed = assignment.commit_refresh_needed
        event_waiter = None
        try:
            while assignment.active:
                commit_refresh_needed.clear()
                success = await self._maybe_refresh_commit_offsets(
                    assignment)

                wait_futures = [assignment.unassign_future]
                if not success:
                    timeout = retry_backoff_ms
                else:
                    timeout = None
                    event_waiter = ensure_future(
                        commit_refresh_needed.wait(), loop=self._loop)
                    wait_futures.append(event_waiter)

                await asyncio.wait(
                    wait_futures,
                    timeout=timeout,
                    return_when=asyncio.FIRST_COMPLETED,
                    loop=self._loop)
        except asyncio.CancelledError:
            pass
        except Exception:
            # Reset event to continue in case user fixes the problem
            commit_refresh_needed.set()
            raise

        # Just to make sure we properly close started tasks we cancel
        # event.wait() task
        if event_waiter is not None and not event_waiter.done():
            event_waiter.cancel()
            event_waiter = None 
示例29
def seek_to_end(self, *partitions):
        """Seek to the most recent available offset for partitions.

        Arguments:
            *partitions: Optionally provide specific TopicPartitions, otherwise
                default to all assigned partitions.

        Raises:
            IllegalStateError: If any partition is not currently assigned
            TypeError: If partitions are not instances of TopicPartition

        .. versionadded:: 0.3.0

        """
        if not all([isinstance(p, TopicPartition) for p in partitions]):
            raise TypeError('partitions must be TopicPartition instances')

        if not partitions:
            partitions = self._subscription.assigned_partitions()
            assert partitions, 'No partitions are currently assigned'
        else:
            not_assigned = (
                set(partitions) - self._subscription.assigned_partitions()
            )
            if not_assigned:
                raise IllegalStateError(
                    "Partitions {} are not assigned".format(not_assigned))

        for tp in partitions:
            log.debug("Seeking to end of partition %s", tp)
        fut = self._fetcher.request_offset_reset(
            partitions, OffsetResetStrategy.LATEST)
        assignment = self._subscription.subscription.assignment
        await asyncio.wait(
            [fut, assignment.unassign_future],
            timeout=self._request_timeout_ms / 1000,
            return_when=asyncio.FIRST_COMPLETED,
            loop=self._loop
        )
        self._coordinator.check_errors()
        return fut.done() 
示例30
def next_request(self) -> bytes:
        getter = self._loop.create_task(self._msgs.get())
        await asyncio.wait(
            [getter, self._con_lost_fut],
            return_when=asyncio.FIRST_COMPLETED)

        if self._con_lost_fut.done():
            getter.cancel()
            return self._con_lost_fut.result()

        return getter.result()