init

Author: mjhea0

.gitignore

@@ -0,0 +1,4 @@
+env
+__pycache__
+*.pyc
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Michael Herman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,28 @@
+## Parallelism, Concurrency, and AsyncIO in Python - by example
+
+Speeding up CPU-bound and IO-bound operations with multiprocessing, threading, and AsyncIO
+
+> Blog post: [Parallelism, Concurrency, and AsyncIO in Python - by example](http://testdriven.io/blog/python-concurrency-parallelism/)
+
+## Setup
+
+1. Fork/Clone
+1. Create and activate a virtual environment
+1. Install the dependencies
+
+## IO-bound Operation
+
+```sh
+$ python io-bound_sync.py
+$ python io-bound_concurrent_1.py
+$ python io-bound_concurrent_2.py
+$ python io-bound_concurrent_3.py
+```
+
+## CPU-bound Operation
+
+```sh
+$ python cpu-bound_sync.py
+$ python cpu-bound_parallel_1.py
+$ python cpu-bound_parallel_2.py
+```

cpu-bound_parallel_1.py

@@ -0,0 +1,22 @@
+# cpu-bound_parallel_1.py
+
+import time
+from multiprocessing import Pool, cpu_count
+
+from tasks import get_prime_numbers
+
+
+def main():
+    with Pool(cpu_count() - 1) as p:
+        p.starmap(get_prime_numbers, zip(range(1000, 16000)))
+        p.close()
+        p.join()
+
+
+if __name__ == "__main__":
+    start_time = time.perf_counter()
+
+    main()
+
+    end_time = time.perf_counter()
+    print(f"Elapsed run time: {end_time - start_time} seconds.")

cpu-bound_parallel_2.py

@@ -0,0 +1,26 @@
+# cpu-bound_parallel_2.py
+
+import time
+from concurrent.futures import ProcessPoolExecutor, wait
+from multiprocessing import cpu_count
+
+from tasks import get_prime_numbers
+
+
+def main():
+    futures = []
+
+    with ProcessPoolExecutor(cpu_count() - 1) as executor:
+        for num in range(1000, 16000):
+            futures.append(executor.submit(get_prime_numbers, num))
+
+    wait(futures)
+
+
+if __name__ == "__main__":
+    start_time = time.perf_counter()
+
+    main()
+
+    end_time = time.perf_counter()
+    print(f"Elapsed run time: {end_time - start_time} seconds.")

cpu-bound_sync.py

@@ -0,0 +1,19 @@
+# cpu-bound_sync.py
+
+import time
+
+from tasks import get_prime_numbers
+
+
+def main():
+    for num in range(1000, 16000):
+        get_prime_numbers(num)
+
+
+if __name__ == "__main__":
+    start_time = time.perf_counter()
+
+    main()
+
+    end_time = time.perf_counter()
+    print(f"Elapsed run time: {end_time - start_time} seconds.")

io-bound_concurrent_1.py

@@ -0,0 +1,26 @@
+# io-bound_concurrent_1.py
+
+import threading
+import time
+
+from tasks import make_request
+
+
+def main():
+    tasks = []
+
+    for num in range(1, 101):
+        tasks.append(threading.Thread(target=make_request, args=(num,)))
+        tasks[-1].start()
+
+    for task in tasks:
+        task.join()
+
+
+if __name__ == "__main__":
+    start_time = time.perf_counter()
+
+    main()
+
+    end_time = time.perf_counter()
+    print(f"Elapsed run time: {end_time - start_time} seconds.")

io-bound_concurrent_2.py

@@ -0,0 +1,25 @@
+# io-bound_concurrent_2.py
+
+import time
+from concurrent.futures import ThreadPoolExecutor, wait
+
+from tasks import make_request
+
+
+def main():
+    futures = []
+
+    with ThreadPoolExecutor() as executor:
+        for num in range(1, 101):
+            futures.append(executor.submit(make_request, num))
+
+    wait(futures)
+
+
+if __name__ == "__main__":
+    start_time = time.perf_counter()
+
+    main()
+
+    end_time = time.perf_counter()
+    print(f"Elapsed run time: {end_time - start_time} seconds.")

io-bound_concurrent_3.py

@@ -0,0 +1,26 @@
+# io-bound_concurrent_3.py
+
+import asyncio
+import time
+
+import httpx
+
+from tasks import make_request_async
+
+
+async def main():
+    async with httpx.AsyncClient() as client:
+        return await asyncio.gather(
+            *[make_request_async(num, client) for num in range(1, 101)]
+        )
+
+
+if __name__ == "__main__":
+    start_time = time.perf_counter()
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(main())
+
+    end_time = time.perf_counter()
+    elapsed_time = end_time - start_time
+    print(f"Elapsed run time: {elapsed_time} seconds")

io-bound_sync.py

@@ -0,0 +1,19 @@
+# io-bound_sync.py
+
+import time
+
+from tasks import make_request
+
+
+def main():
+    for num in range(1, 101):
+        make_request(num)
+
+
+if __name__ == "__main__":
+    start_time = time.perf_counter()
+
+    main()
+
+    end_time = time.perf_counter()
+    print(f"Elapsed run time: {end_time - start_time} seconds.")

requirements.txt

@@ -0,0 +1,2 @@
+httpx==0.15.5
+requests==2.24.0

tasks.py

@@ -0,0 +1,58 @@
+# tasks.py
+
+import os
+from multiprocessing import current_process
+from threading import current_thread
+
+import requests
+
+
+def make_request(num):
+    # io-bound
+
+    pid = os.getpid()
+    thread_name = current_thread().name
+    process_name = current_process().name
+    print(f"{pid} - {process_name} - {thread_name}")
+
+    requests.get("https://httpbin.org/ip")
+
+
+async def make_request_async(num, client):
+    # io-bound
+
+    pid = os.getpid()
+    thread_name = current_thread().name
+    process_name = current_process().name
+    print(f"{pid} - {process_name} - {thread_name}")
+
+    await client.get("https://httpbin.org/ip")
+
+
+def get_prime_numbers(num):
+    # cpu-bound
+
+    pid = os.getpid()
+    thread_name = current_thread().name
+    process_name = current_process().name
+    print(f"{pid} - {process_name} - {thread_name}")
+
+    numbers = []
+
+    prime = [True for i in range(num + 1)]
+    p = 2
+
+    while p * p <= num:
+        if prime[p]:
+            for i in range(p * 2, num + 1, p):
+                prime[i] = False
+        p += 1
+
+    prime[0] = False
+    prime[1] = False
+
+    for p in range(num + 1):
+        if prime[p]:
+            numbers.append(p)
+
+    return numbers