There is this package. I have not used it myself. I also am a big fan of celery for this kind of work. I have used this for a long time and am comfortable with it.
Or … you could roll something yourself like this:
import asyncio
import signal
from datetime import datetime
MAXSIZE = 0 # Set to 0 to disable, or anything larger to create a limit
NUM_WORKERS = 3
INTERVAL = 3 # number of seconds
class Worker:
def __init__(self, name, queue):
self.name = name
self.queue = queue
print(f"{name} online and ready to work")
async def run(self):
while True:
job = await self.queue.get()
print(f"{self.name} received a job: {job}")
asyncio.create_task(self.process(job))
await asyncio.sleep(1)
async def process(self, job):
print(f"{self.name} processing {job}")
class Fetcher:
def __init__(self, queue):
self.queue = queue
async def run(self):
while True:
await self.fetch()
await asyncio.sleep(INTERVAL)
async def fetch(self):
print('Fetching from service')
t = datetime.now().isoformat()
task = f'DUMMY INSTRUCTIONS ({t})'
await self.queue.put(task)
def start(workers=1):
loop = asyncio.get_event_loop()
queue = asyncio.Queue(loop=loop, maxsize=MAXSIZE)
fetcher = Fetcher(queue)
loop.create_task(fetcher.run())
for x in range(workers):
worker = Worker(f"Worker-{x}", queue)
loop.create_task(worker.run())
signal.signal(signal.SIGINT, signal.SIG_DFL)
try:
loop.run_forever()
except KeyboardInterrupt:
asyncio.gather(*asyncio.Task.all_tasks()).cancel()
loop.stop()
loop.close()
if __name__ == '__main__':
start(workers=NUM_WORKERS)
Basically is creates a task for a Fetcher to do your long polling (send an outbound 443 request every 10 minutes). It also creates a number of Worker instances. When the fetcher sees a new set of instructions, it pushes it to a queue. Each of the workers listens to that queue, and when it sees something to be done, it sends it to a new task to be processed.
Running this, you should see something like:
$ python c.py
Worker-0 online and ready to work
Worker-1 online and ready to work
Worker-2 online and ready to work
Fetching from service
Worker-0 received a job: DUMMY INSTRUCTIONS (2018-12-31T10:19:35.314996)
Worker-0 processing DUMMY INSTRUCTIONS (2018-12-31T10:19:35.314996)
Fetching from service
Worker-1 received a job: DUMMY INSTRUCTIONS (2018-12-31T10:19:38.317484)
Worker-1 processing DUMMY INSTRUCTIONS (2018-12-31T10:19:38.317484)
Fetching from service
Worker-2 received a job: DUMMY INSTRUCTIONS (2018-12-31T10:19:41.320152)
Worker-2 processing DUMMY INSTRUCTIONS (2018-12-31T10:19:41.320152)
Fetching from service
Worker-0 received a job: DUMMY INSTRUCTIONS (2018-12-31T10:19:44.322922)
Worker-0 processing DUMMY INSTRUCTIONS (2018-12-31T10:19:44.322922)
Fetching from service
Worker-1 received a job: DUMMY INSTRUCTIONS (2018-12-31T10:19:47.325449)
Worker-1 processing DUMMY INSTRUCTIONS (2018-12-31T10:19:47.325449)
Fetching from service
Worker-2 received a job: DUMMY INSTRUCTIONS (2018-12-31T10:19:50.328079)
Worker-2 processing DUMMY INSTRUCTIONS (2018-12-31T10:19:50.328079)
Fetching from service
Worker-0 received a job: DUMMY INSTRUCTIONS (2018-12-31T10:19:53.329973)
Worker-0 processing DUMMY INSTRUCTIONS (2018-12-31T10:19:53.329973)
