Python源码示例:asyncio.Semaphore()
示例1
def interval_task(self):
try:
to_backup = self.bot.db.intervals.find({"next": {
"$lt": datetime.utcnow()
}})
semaphore = Semaphore(10)
async for interval in to_backup:
async def run_interval():
try:
next = datetime.utcnow() + timedelta(minutes=interval["interval"])
await self.bot.db.intervals.update_one({"_id": interval["_id"]}, {"$set": {"next": next}})
await self.run_backup(interval["_id"])
finally:
semaphore.release()
await semaphore.acquire()
self.bot.loop.create_task(run_interval())
await sleep(0)
except Exception:
pass
示例2
def main(global_delay, local_delay, concurrency):
global global_sleep, local_sleep, semaphore, index
global_sleep = global_delay
local_sleep = local_delay
semaphore = asyncio.Semaphore(concurrency)
print('Global delay =', global_delay)
print('Local delay =', local_delay)
print('Max. concurrency =', concurrency)
print('Building inverted index...')
index = build_index()
app = web.Application()
app.router.add_get('/', usage)
app.router.add_get('/index/{word}', index_for)
app.router.add_get('/name/{char}', char_name)
print('Listening on port', PORT)
web.run_app(app, port=PORT)
示例3
def downloader_coro(loop, cc_list, base_url, verbose, concur_req):
counter = collections.Counter()
semaphore = asyncio.Semaphore(concur_req)
async with aiohttp.ClientSession(loop=loop) as client:
to_do = [download_one(client, cc, base_url, semaphore, verbose)
for cc in sorted(cc_list)]
to_do_iter = asyncio.as_completed(to_do)
if not verbose:
to_do_iter = tqdm.tqdm(to_do_iter, total=len(cc_list))
for future in to_do_iter:
try:
res = await future
except FetchError as exc:
country_code = exc.country_code
try:
error_msg = exc.__cause__.args[0]
except IndexError:
error_msg = exc.__cause__.__class__.__name__
if verbose and error_msg:
msg = '*** Error for {}: {}'
print(msg.format(country_code, error_msg))
status = HTTPStatus.error
else:
status = res.status
counter[status] += 1
return counter
示例4
def __init__(self, in_state, in_forward, search_threads):
self.noise_eps = 0.25
self.dirichlet_alpha = 0.3 #0.03
self.p_ = (1 - self.noise_eps) * 1 + self.noise_eps * np.random.dirichlet([self.dirichlet_alpha])
self.root = leaf_node(None, self.p_, in_state)
self.c_puct = 5 #1.5
# self.policy_network = in_policy_network
self.forward = in_forward
self.node_lock = defaultdict(Lock)
self.virtual_loss = 3
self.now_expanding = set()
self.expanded = set()
self.cut_off_depth = 30
# self.QueueItem = namedtuple("QueueItem", "feature future")
self.sem = asyncio.Semaphore(search_threads)
self.queue = Queue(search_threads)
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
示例5
def __init__(self, in_state, in_forward, search_threads):
self.noise_eps = 0.25
self.dirichlet_alpha = 0.3 #0.03
self.p_ = (1 - self.noise_eps) * 1 + self.noise_eps * np.random.dirichlet([self.dirichlet_alpha])
self.root = leaf_node(None, self.p_, in_state)
self.c_puct = 5 #1.5
# self.policy_network = in_policy_network
self.forward = in_forward
self.node_lock = defaultdict(Lock)
self.virtual_loss = 3
self.now_expanding = set()
self.expanded = set()
self.cut_off_depth = 30
# self.QueueItem = namedtuple("QueueItem", "feature future")
self.sem = asyncio.Semaphore(search_threads)
self.queue = Queue(search_threads)
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
示例6
def _check_uris_async(
links_to_check: Iterable[str], max_threads: int = 10, delay: float = 0
) -> Iterable[UrlResult]:
tasks = []
# create instance of Semaphore
sem = asyncio.Semaphore(max_threads)
# Create client session that will ensure we dont open new connection
# per each request.
async with ClientSession() as session:
for uri in links_to_check:
if delay:
asyncio.sleep(delay)
# pass Semaphore and session to every GET request
task = asyncio.ensure_future(_check_uri_with_sem_async(sem, uri, session))
tasks.append(task)
results = await asyncio.gather(*tasks)
return results
示例7
def __init__(self, threads: int = None) -> None:
try: # get or create loop (threads don't have one)
#: our asyncio loop
self.loop = asyncio.get_event_loop()
except RuntimeError:
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
#: number of threads to use
self.threads = threads or threads_to_use()
#: semaphore to limit io parallelism
self.io_sem: asyncio.Semaphore = asyncio.Semaphore(1)
#: must never run more than one conda at the same time
#: (used by PyPi when running skeleton)
self.conda_sem: asyncio.Semaphore = asyncio.Semaphore(1)
#: the filters successively applied to each item
self.filters: List[AsyncFilter] = []
#: executor running things in separate python processes
self.proc_pool_executor = ProcessPoolExecutor(self.threads)
self._shutting_down = False
示例8
def __init__(
self,
context: InjectionContext,
receive_inbound: Coroutine,
return_inbound: Callable = None,
):
"""Initialize an `InboundTransportManager` instance."""
self.context = context
self.max_message_size = 0
self.receive_inbound = receive_inbound
self.return_inbound = return_inbound
self.registered_transports = {}
self.running_transports = {}
self.sessions = OrderedDict()
self.session_limit: asyncio.Semaphore = None
self.task_queue = TaskQueue()
self.undelivered_queue: DeliveryQueue = None
示例9
def setup(self):
"""Perform setup operations."""
# Load config settings
if self.context.settings.get("transport.max_message_size"):
self.max_message_size = self.context.settings["transport.max_message_size"]
inbound_transports = (
self.context.settings.get("transport.inbound_configs") or []
)
for transport in inbound_transports:
module, host, port = transport
self.register(
InboundTransportConfiguration(module=module, host=host, port=port)
)
# Setup queue for undelivered messages
if self.context.settings.get("transport.enable_undelivered_queue"):
self.undelivered_queue = DeliveryQueue()
# self.session_limit = asyncio.Semaphore(50)
示例10
def __init__(self, daemon, coin, blocks_event):
self.logger = class_logger(__name__, self.__class__.__name__)
self.daemon = daemon
self.coin = coin
self.blocks_event = blocks_event
self.blocks = []
self.caught_up = False
# Access to fetched_height should be protected by the semaphore
self.fetched_height = None
self.semaphore = asyncio.Semaphore()
self.refill_event = asyncio.Event()
# The prefetched block cache size. The min cache size has
# little effect on sync time.
self.cache_size = 0
self.min_cache_size = 10 * 1024 * 1024
# This makes the first fetch be 10 blocks
self.ave_size = self.min_cache_size // 10
self.polling_delay = 5
示例11
def scan_host(address, start_port, end_port, max_conns=400):
"""
:param address: IPv4 address to scan
:param start_port: First port value to scan
:param end_port: Last port value to scan
:param max_conns: Maximum simultaneous number of connections
:return:
"""
sem = asyncio.Semaphore(max_conns)
ports = range(start_port, end_port)
tasks = [asyncio.ensure_future(PortScanner.check_port_sem(sem, address, port)) for port in ports]
responses = await asyncio.gather(*tasks)
open_ports = list(filter(lambda x: x is not None, responses))
return open_ports
示例12
def find_node_ports(address, ports):
"""
Find all the open ports for a host.
:param address: IP address of the host.
:param ports: Port to check.
:return: A list of the found open ports.
"""
sem = asyncio.Semaphore(400) # Change this value for concurrency limitation
tasks = [asyncio.ensure_future(check_port_sem(sem, address, p)) for p in ports]
found_ports = []
for response in await asyncio.gather(*tasks):
if response:
found_ports.append(response)
return found_ports
示例13
def check_routers(self, addresses: List[str], ports: List[int]) -> List[BaseIndustrialRouter]:
"""
Check for routers in a range of addressess and ports.
:param addresses: List of addressess to be checked.
:param ports: List of ports to be checked for each address.
:return: A list of found routers.
"""
async def check_routers_aio(addresses, ports):
semaphore = Semaphore(50)
futures = []
routers = []
for address in addresses:
for port in ports:
futures.append(asyncio.ensure_future(self.check_is_router(address, port, semaphore=semaphore)))
done, pending = await asyncio.wait(futures)
for future in done:
if future:
routers.append(future)
return routers
return asyncio.run(check_routers_aio(addresses, ports), debug=True)
示例14
def check_router_credentials(self, routers: List[BaseIndustrialRouter]):
"""
Check default credentials for a list of routers.
:param routers: List of routers to be checked.
"""
async def check_router_credentials_aio(routers):
semaphore = Semaphore(100)
futures = []
for router in routers:
if isinstance(router, self.__class__.router_cls):
futures.append(asyncio.ensure_future(self.check_default_password(router, semaphore=semaphore)))
await asyncio.wait(futures, return_when=ALL_COMPLETED, )
asyncio.run(check_router_credentials_aio(routers), debug=True)
示例15
def test_context_manager_async_with(self):
primitives = [
asyncio.Lock(loop=self.loop),
asyncio.Condition(loop=self.loop),
asyncio.Semaphore(loop=self.loop),
asyncio.BoundedSemaphore(loop=self.loop),
]
async def test(lock):
await asyncio.sleep(0.01, loop=self.loop)
self.assertFalse(lock.locked())
async with lock as _lock:
self.assertIs(_lock, None)
self.assertTrue(lock.locked())
await asyncio.sleep(0.01, loop=self.loop)
self.assertTrue(lock.locked())
self.assertFalse(lock.locked())
for primitive in primitives:
self.loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
示例16
def test_context_manager_with_await(self):
primitives = [
asyncio.Lock(loop=self.loop),
asyncio.Condition(loop=self.loop),
asyncio.Semaphore(loop=self.loop),
asyncio.BoundedSemaphore(loop=self.loop),
]
async def test(lock):
await asyncio.sleep(0.01, loop=self.loop)
self.assertFalse(lock.locked())
with await lock as _lock:
self.assertIs(_lock, None)
self.assertTrue(lock.locked())
await asyncio.sleep(0.01, loop=self.loop)
self.assertTrue(lock.locked())
self.assertFalse(lock.locked())
for primitive in primitives:
self.loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
示例17
def test_semaphore(self):
sem = asyncio.Semaphore(loop=self.loop)
self.assertEqual(1, sem._value)
@asyncio.coroutine
def acquire_lock():
return (yield from sem)
res = self.loop.run_until_complete(acquire_lock())
self.assertTrue(res)
self.assertTrue(sem.locked())
self.assertEqual(0, sem._value)
sem.release()
self.assertFalse(sem.locked())
self.assertEqual(1, sem._value)
示例18
def test_acquire_cancel_before_awoken(self):
sem = asyncio.Semaphore(value=0, loop=self.loop)
t1 = asyncio.Task(sem.acquire(), loop=self.loop)
t2 = asyncio.Task(sem.acquire(), loop=self.loop)
t3 = asyncio.Task(sem.acquire(), loop=self.loop)
t4 = asyncio.Task(sem.acquire(), loop=self.loop)
test_utils.run_briefly(self.loop)
sem.release()
t1.cancel()
t2.cancel()
test_utils.run_briefly(self.loop)
num_done = sum(t.done() for t in [t3, t4])
self.assertEqual(num_done, 1)
t3.cancel()
t4.cancel()
test_utils.run_briefly(self.loop)
示例19
def __init__(self, config: Config, model, play_config=None):
self.config = config
self.model = model
self.play_config = play_config or self.config.play
self.api = Connect4ModelAPI(self.config, self.model)
self.labels_n = config.n_labels
self.var_n = defaultdict(lambda: np.zeros((self.labels_n,)))
self.var_w = defaultdict(lambda: np.zeros((self.labels_n,)))
self.var_q = defaultdict(lambda: np.zeros((self.labels_n,)))
self.var_u = defaultdict(lambda: np.zeros((self.labels_n,)))
self.var_p = defaultdict(lambda: np.zeros((self.labels_n,)))
self.expanded = set()
self.now_expanding = set()
self.prediction_queue = Queue(self.play_config.prediction_queue_size)
self.sem = asyncio.Semaphore(self.play_config.parallel_search_num)
self.moves = []
self.loop = asyncio.get_event_loop()
self.running_simulation_num = 0
self.thinking_history = {} # for fun
示例20
def test_repr(self):
sem = asyncio.Semaphore(loop=self.loop)
self.assertTrue(repr(sem).endswith('[unlocked,value:1]>'))
self.assertTrue(RGX_REPR.match(repr(sem)))
self.loop.run_until_complete(sem.acquire())
self.assertTrue(repr(sem).endswith('[locked]>'))
self.assertTrue('waiters' not in repr(sem))
self.assertTrue(RGX_REPR.match(repr(sem)))
sem._waiters.append(mock.Mock())
self.assertTrue('waiters:1' in repr(sem))
self.assertTrue(RGX_REPR.match(repr(sem)))
sem._waiters.append(mock.Mock())
self.assertTrue('waiters:2' in repr(sem))
self.assertTrue(RGX_REPR.match(repr(sem)))
示例21
def test_semaphore(self):
sem = asyncio.Semaphore(loop=self.loop)
self.assertEqual(1, sem._value)
@asyncio.coroutine
def acquire_lock():
return (yield from sem)
res = self.loop.run_until_complete(acquire_lock())
self.assertTrue(res)
self.assertTrue(sem.locked())
self.assertEqual(0, sem._value)
sem.release()
self.assertFalse(sem.locked())
self.assertEqual(1, sem._value)
示例22
def test_acquire_cancel_before_awoken(self):
sem = asyncio.Semaphore(value=0, loop=self.loop)
t1 = asyncio.Task(sem.acquire(), loop=self.loop)
t2 = asyncio.Task(sem.acquire(), loop=self.loop)
t3 = asyncio.Task(sem.acquire(), loop=self.loop)
t4 = asyncio.Task(sem.acquire(), loop=self.loop)
test_utils.run_briefly(self.loop)
sem.release()
t1.cancel()
t2.cancel()
test_utils.run_briefly(self.loop)
num_done = sum(t.done() for t in [t3, t4])
self.assertEqual(num_done, 1)
t3.cancel()
t4.cancel()
test_utils.run_briefly(self.loop)
示例23
def print_tests():
st = '''Available functions:
print_tests() Print this list.
ack_test() Test event acknowledge and Message class.
message_test() Test Message class.
event_test() Test Event and Lock objects.
barrier_test() Test the Barrier class.
semaphore_test(bounded=False) Test Semaphore or BoundedSemaphore.
condition_test() Test the Condition class.
queue_test() Test the Queue class
Recommended to issue ctrl-D after running each test.
'''
print('\x1b[32m')
print(st)
print('\x1b[39m')
示例24
def _fill_body(body: Body, semaphore: asyncio.Semaphore, limit: int) -> None:
for number in range(limit):
body.append(b"%d" % number)
await semaphore.acquire()
body.set_complete()
示例25
def test_full_body() -> None:
body = Body(None, None)
limit = 3
semaphore = asyncio.Semaphore(limit)
asyncio.ensure_future(_fill_body(body, semaphore, limit))
assert b"012" == await body # type: ignore
示例26
def test_body_streaming() -> None:
body = Body(None, None)
limit = 3
semaphore = asyncio.Semaphore(0)
asyncio.ensure_future(_fill_body(body, semaphore, limit))
index = 0
async for data in body: # type: ignore
semaphore.release()
assert data == b"%d" % index
index += 1
assert b"" == await body # type: ignore
示例27
def test_body_streaming_no_data() -> None:
body = Body(None, None)
semaphore = asyncio.Semaphore(0)
asyncio.ensure_future(_fill_body(body, semaphore, 0))
async for _ in body: # type: ignore # noqa: F841
assert False # Should not reach this
assert b"" == await body # type: ignore
示例28
def init(sanic, loop):
global sem
concurrency_per_worker = 4
sem = asyncio.Semaphore(concurrency_per_worker, loop=loop)
示例29
def __init__(self, cb):
super().__init__()
self.cb = cb
self.cnt= 0
self.log = logging.getLogger() #.getChild('TcpConnection')
self.log.info('TcpConnection: creating TcpCOnnection')
self.queue = asyncio.Queue(maxsize=100)
self.hipri_queue = asyncio.Queue()
self._ready = asyncio.Event()
self._msg_ready = asyncio.Semaphore(value=0)
self.tsk= asyncio.async(self._send_messages()) # Or asyncio.ensure_future if using 3.4.3+
self.flush= False
示例30
def run(loop, r):
url = config.get("server_url") + "{}"
tasks = []
# create instance of Semaphore
batch_size = int(config.get("batch_size"))
sem = asyncio.Semaphore(batch_size)
for i in range(r):
# pass Semaphore to every GET request
task = asyncio.ensure_future(bound_fetch(sem, url.format(i)))
tasks.append(task)
responses = await asyncio.gather(*tasks)
#await responses
print("Got {} responses".format(len(responses)))