wsgi tuning setup config (#1947)

* will cleanup a little more before final

* cleanup

* EOF

* instructions update

* Small doc update

Author: JenniWhitman

.gitignore

@@ -126,3 +126,4 @@ celerybeat-schedule
 docker-compose.*.yml
 
 .python-version
+/locustfile.py

docs/source/configuration/uwsgi_tuning.rst

@@ -0,0 +1,68 @@
+=======================================
+Setting up uWsgi tuning for MITx Online
+=======================================
+
+This setup satisfies the testing to help with tuning as mentioned in this `Discusssion Post <https://github.com/mitodl/hq/discussions/393>`_
+
+Largely borrowed from work on OCW studio:
+
+| `Adding uWSGI stats <https://github.com/mitodl/ocw-studio/pull/1898/>`_
+| `Tuning the App <https://github.com/mitodl/ocw-studio/pull/1886/>`_
+
+
+******************
+To set up locally:
+******************
+
+Set up uwsgitop
+---------------
+1. Install uwsgitop: ``docker compose run --rm web poetry add uwsgitop``
+2. Set UWSGI_RELOAD_ON_RSS in your .env to a high value (e.g. 500)
+3. Set UWSGI_MAX_REQUESTS in your .env to a high value (e.g. 10000)
+4. ``docker compose build``
+5. ``docker compose up``
+6. In a new terminal window/tab, ``docker compose exec web uwsgitop /tmp/uwsgi-stats.sock``
+7. You should see your application's memory usage without usage. Ready to go.
+
+
+Set up Locust
+-------------
+1. Install Locust: ``docker compose run --rm web poetry add locust``
+2. Add locust to your docker-compose.yml locally, under services:
+
+.. code-block:: shell
+
+	locust:
+	  image: locustio/locust
+	  ports:
+	    - "8089:8089"
+	  volumes:
+	    - ./:/src
+	  command: >
+	    -f /src/locustfile.py
+
+3. Add the following to the web block, at the level of, and directly after, ``build``:
+
+.. code-block:: shell
+
+    deploy:
+      resources:
+        limits:
+          cpus: "2"
+          memory: "1g"
+
+4. Add locustfile.py. There is an example file at ``locustfile.py.example`` in the root of the repo.  ``cp locustfile.py.example locustfile.py`` will copy it over as is. Change variables and/or add tests as needed.
+
+Put it all together
+-------------------
+
+1. Run ``docker-compose build``
+2. Run ``docker-compose up``
+3. You can use locust from ``http://0.0.0.0:8089/`` in a browser
+4. You can use uwsgitop in a terminal with ``docker compose exec web uwsgitop /tmp/uwsgi-stats.sock``
+
+******************
+To test:
+******************
+
+Coming soon!

locustfile.py.example

@@ -0,0 +1,92 @@
+import logging
+from locust import HttpUser, task, between, events
+
+
+def _get_product_json(response_json, page_type):
+    logging.info(f'User was displayed {response_json["count"]} {page_type}(s)')
+    pages = response_json["results"][:5]
+    return [page["readable_id"] for page in pages]
+
+
+class DjangoAdminUser(HttpUser):
+    wait_time = between(1, 5)
+
+    def on_start(self):
+        self.client.get("/admin/login/")
+        self.login()
+
+    def login(self):
+        csrf_token = self.client.cookies['csrftoken']
+        self.client.post(
+            "/admin/login/",
+            {
+                "username": "[email protected]",
+                "password": "edx",
+                "csrfmiddlewaretoken": csrf_token,
+            },
+            headers={"Referer": "/admin/login/"},
+        )
+
+    @task
+    def navigate_through_course_areas(self):
+        logging.info(f"{self} has started to surf")
+        self._navigate_to_home_page()
+        programs, courses = self._navigate_to_catalog()
+        self._navigate_to_product_page("program", programs)
+        self._navigate_to_product_page("course", courses)
+        self._navigate_to_checkout_page()
+        logging.info(f"{self} has gone off to learn")
+
+    def _navigate_to_catalog(self):
+        endpoints = [
+            "/api/users/me",
+            "/api/departments/",
+            "/api/users/me",
+        ]
+        logging.info(f"User {self} opening catalog")
+        self._navigate_to_endpoints(endpoints)
+        with self.client.get("/api/programs/?page=1&live=true", catch_response=True) as response:
+            program_json = _get_product_json(response.json(), "program")
+        with self.client.get("/api/courses/?page=1&live=true&page__live=true&courserun_is_enrollable=true",
+                             catch_response=True) as response:
+            course_json = _get_product_json(response.json(), "course")
+        return program_json, course_json
+
+    def _navigate_to_home_page(self):
+        logging.info("User (%r) opening Home Page")
+        with self.client.get("/api/users/me", catch_response=True) as response:
+            logging.info(response.status_code)
+
+    def _navigate_to_checkout_page(self):
+        logging.info("User (%r) opening Checkout Page")
+        endpoints = [
+            "/api/users/me",
+            "/api/checkout/cart",
+        ]
+        self._navigate_to_endpoints(endpoints)
+
+    def _navigate_to_product_page(self, product_type, response_json):
+        logging.info(response_json)
+        for page in response_json:
+            logging.info(f"User opened {product_type} page for {page}")
+            endpoints = [
+                "/api/users/me",
+                "/api/program_enrollments/",
+                "/api/users/me",
+                f"/api/course_runs/?relevant_to={page}",
+            ]
+            if product_type == "course":
+                endpoints += [
+                    f"/api/courses/?readable_id={page}&live=true",
+                ]
+            elif product_type == "program":
+                endpoints += [
+                    f"/api/programs/?readable_id={page}",
+                ]
+            self._navigate_to_endpoints(endpoints)
+
+    def _navigate_to_endpoints(self, endpoints):
+        for endpoint in endpoints:
+            logging.info(f"endpoint: {endpoint}")
+            with self.client.get(endpoint, catch_response=True) as response:
+                logging.info(f"status: {response.status_code}")

poetry.lock

@@ -83,6 +83,7 @@ uwsgi = "^2.0.19"
 wagtail = "^5.0"
 hypothesis = "4.23.9"
 posthog = "^3.0.1"
+uwsgitop = "^0.11"
 
 
 [tool.poetry.group.dev.dependencies]

pyproject.toml

@@ -45,3 +45,4 @@ endif =
 if-not-env = UWSGI_POST_BUFFERING
 post-buffering = 65535
 endif =
+stats = /tmp/uwsgi-stats.sock