Make not slow (#456)

* bump stuff

* add type hints to ldap.py

* more sane ldap queries

* improved gatekeep calculations

* improve housing queue calculations

* reorganize directorships calculations

* fix calculations and remove old function

* fix linting

* oops fix typo

* squish lines together

* speedup spring evals page

* More spring evals speedup

* fix linting

* speed up intro evals

* profiling documentation

* added profiling env variable

* oops i missed a part

---------

Co-authored-by: Noah Hanford (spaced) <spaced@csh.rit.edu>

Author: BigSpaceships

README.md

@@ -61,6 +61,8 @@ This will run the asset pipeline, start the Python server, and start BrowserSync
 
 To add new dependencies, add them to `requirements.in` and then run `pip-compile requirements.in` to produce a new locked `requirements.txt`. Do not edit `requirements.txt` directly as it will be overwritten by future PRs.
 
+after app initialization
+
 ### Database Migrations
 
 If the database schema is changed after initializing the database, you must migrate it to the new schema by running:

conditional/__init__.py

@@ -13,6 +13,7 @@
 import sentry_sdk
 from sentry_sdk.integrations.flask import FlaskIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
+from werkzeug.middleware.profiler import ProfilerMiddleware
 
 app = Flask(__name__)
 gzip = Gzip(app)
@@ -29,6 +30,12 @@
 db = SQLAlchemy(app)
 migrate = Migrate(app, db)
 
+if app.config['PROFILING']:
+    app.wsgi_app = ProfilerMiddleware(
+        app.wsgi_app,
+        restrictions=[30]
+    )
+
 # Sentry setup
 sentry_sdk.init(
     dsn=app.config['SENTRY_DSN'],

conditional/blueprints/dashboard.py

@@ -36,6 +36,7 @@ def display_dashboard(user_dict=None):
     log.info('display dashboard')
 
     # Get the list of voting members.
+
     can_vote = get_voting_members()
 
     data = {}
@@ -56,7 +57,7 @@ def display_dashboard(user_dict=None):
     spring = {}
     c_meetings = get_cm(user_dict['account'])
     spring['committee_meetings'] = len(c_meetings)
-    spring['req_meetings'] = req_cm(user_dict['account'])
+    spring['req_meetings'] = req_cm(user_dict['account'].uid)
     h_meetings = [(m.meeting_id, m.attendance_status) for m in get_hm(user_dict['account'])]
     spring['hm_missed'] = len([h for h in h_meetings if h[1] == "Absent"])
     eval_entry = SpringEval.query.filter(SpringEval.uid == user_dict['account'].uid,

conditional/blueprints/intro_evals.py

@@ -2,6 +2,7 @@
 
 import structlog
 from flask import Blueprint, request
+from sqlalchemy import func
 
 from conditional import start_of_year, auth
 from conditional.models.models import CommitteeMeeting
@@ -11,89 +12,182 @@
 from conditional.models.models import FreshmanHouseMeetingAttendance
 from conditional.models.models import FreshmanSeminarAttendance
 from conditional.models.models import HouseMeeting
+from conditional.models.models import MemberCommitteeAttendance
 from conditional.models.models import MemberHouseMeetingAttendance
 from conditional.models.models import MemberSeminarAttendance
 from conditional.models.models import TechnicalSeminar
 from conditional.util.auth import get_user
 from conditional.util.flask import render_template
 from conditional.util.ldap import ldap_get_intro_members
-from conditional.util.member import get_cm, get_hm
 
 intro_evals_bp = Blueprint('intro_evals_bp', __name__)
 
 logger = structlog.get_logger()
 
-
-@intro_evals_bp.route('/intro_evals/')
-@auth.oidc_auth("default")
-@get_user
-def display_intro_evals(internal=False, user_dict=None):
-    log = logger.new(request=request, auth_dict=user_dict)
-    log.info('Display Intro Evals Listing')
-
-    # get user data
-    def get_fid_cm_count(member_id):
-        return len([a for a in FreshmanCommitteeAttendance.query.filter(
-            FreshmanCommitteeAttendance.fid == member_id)
-            if CommitteeMeeting.query.filter(CommitteeMeeting.id == a.meeting_id).first().approved])
-
-    members = ldap_get_intro_members()
-
-    ie_members = []
+def get_intro_members_without_accounts():
+    freshman_cm_count = dict([tuple(row) for row in FreshmanCommitteeAttendance.query.join(
+        CommitteeMeeting,
+        FreshmanCommitteeAttendance.meeting_id == CommitteeMeeting.id
+    ).with_entities(
+        FreshmanCommitteeAttendance.fid,
+        CommitteeMeeting.timestamp,
+        CommitteeMeeting.approved,
+    ).filter(
+        CommitteeMeeting.approved,
+        CommitteeMeeting.timestamp >= start_of_year()
+    ).with_entities(
+        FreshmanCommitteeAttendance.fid,
+        func.count(FreshmanCommitteeAttendance.fid) #pylint: disable=not-callable
+    ).group_by(
+        FreshmanCommitteeAttendance.fid
+    ).all()])
+
+    freshman_hm_missed = dict([tuple(row) for row in FreshmanHouseMeetingAttendance.query.join(
+        HouseMeeting,
+        FreshmanHouseMeetingAttendance.meeting_id == HouseMeeting.id
+    ).filter(
+        HouseMeeting.date >= start_of_year(),
+        FreshmanHouseMeetingAttendance.attendance_status == 'Absent'
+    ).with_entities(
+        FreshmanHouseMeetingAttendance.fid,
+            func.count(FreshmanHouseMeetingAttendance.fid) #pylint: disable=not-callable
+    ).group_by(
+        FreshmanHouseMeetingAttendance.fid
+    ).all()])
+
+    freshman_ts_attendance_query = FreshmanSeminarAttendance.query.join(
+        TechnicalSeminar,
+        FreshmanSeminarAttendance.seminar_id == TechnicalSeminar.id
+    ).with_entities(
+        FreshmanSeminarAttendance.fid,
+        TechnicalSeminar.timestamp,
+        TechnicalSeminar.approved
+    ).filter(
+        TechnicalSeminar.approved,
+        TechnicalSeminar.timestamp >= start_of_year()
+    ).with_entities(
+        FreshmanSeminarAttendance.fid,
+        TechnicalSeminar.name
+    ).all()
+
+    freshman_ts_attendance_dict = {}
+
+    for row in freshman_ts_attendance_query:
+        if not row[0] in freshman_ts_attendance_dict:
+            freshman_ts_attendance_dict[row[0]] = []
+
+            freshman_ts_attendance_dict[row[0]].append(row[1])
 
     # freshmen who don't have accounts
-    fids = list(FreshmanAccount.query.filter(
+    freshman_accounts = list(FreshmanAccount.query.filter(
         FreshmanAccount.eval_date > start_of_year(),
         FreshmanAccount.eval_date > datetime.now()))
 
-    for fid in fids:
-        h_meetings = [m.meeting_id for m in
-                      FreshmanHouseMeetingAttendance.query.filter(
-                          FreshmanHouseMeetingAttendance.fid == fid.id
-                      ).filter(
-                          FreshmanHouseMeetingAttendance.attendance_status == "Absent"
-                      )]
+    ie_members = []
 
-        if fid.signatures_missed is None:
+    for freshman_account in freshman_accounts:
+        missed_hms = []
+        if freshman_hm_missed.get(freshman_account.id, 0) != 0:
+            missed_hms = FreshmanHouseMeetingAttendance.query.join(
+                    HouseMeeting,
+                    FreshmanHouseMeetingAttendance.meeting_id == HouseMeeting.id
+            ).filter(
+                    HouseMeeting.date >= start_of_year(), # TODO: this needs to be fixed
+                    FreshmanHouseMeetingAttendance.attendance_status == 'Absent',
+                    FreshmanHouseMeetingAttendance.fid == freshman_account.id,
+            ).with_entities(
+                    func.array_agg(HouseMeeting.date)
+            ).scalar()
+
+        if freshman_account.signatures_missed is None:
             signatures_missed = -1
         else:
-            signatures_missed = fid.signatures_missed
+            signatures_missed = freshman_account.signatures_missed
+
+        cms_attended = freshman_cm_count.get(freshman_account.id, 0)
 
         freshman = {
-            'name': fid.name,
-            'uid': fid.id,
-            'eval_date': fid.eval_date.strftime("%Y-%m-%d"),
+            'name': freshman_account.name,
+            'uid': freshman_account.id,
+            'eval_date': freshman_account.eval_date.strftime("%Y-%m-%d"),
             'signatures_missed': signatures_missed,
-            'committee_meetings': get_fid_cm_count(fid.id),
-            'committee_meetings_passed': get_fid_cm_count(fid.id) >= 6,
-            'house_meetings_missed':
-                [
-                    {
-                        "date": m.date.strftime("%Y-%m-%d"),
-                        "reason":
-                            FreshmanHouseMeetingAttendance.query.filter(
-                                FreshmanHouseMeetingAttendance.fid == fid.id).filter(
-                                FreshmanHouseMeetingAttendance.meeting_id == m.id).first().excuse
-                    }
-                    for m in HouseMeeting.query.filter(
-                        HouseMeeting.id.in_(h_meetings)
-                    )
-                ],
-            'technical_seminars':
-                [s.name for s in TechnicalSeminar.query.filter(
-                    TechnicalSeminar.id.in_(
-                        [a.seminar_id for a in FreshmanSeminarAttendance.query.filter(
-                            FreshmanSeminarAttendance.fid == fid.id)
-                            if TechnicalSeminar.query.filter(TechnicalSeminar.id == a.seminar_id).first().approved]
-                    ))
-                 ],
+            'committee_meetings': cms_attended,
+            'committee_meetings_passed': cms_attended >= 6,
+            'house_meetings_missed': missed_hms,
+            'technical_seminars': freshman_ts_attendance_dict.get(freshman_account.id, []),
             'social_events': '',
             'comments': "",
             'ldap_account': False,
             'status': "Pending"
         }
         ie_members.append(freshman)
 
+    return ie_members
+
+@intro_evals_bp.route('/intro_evals/')
+@auth.oidc_auth("default")
+@get_user
+def display_intro_evals(internal=False, user_dict=None):
+    log = logger.new(request=request, auth_dict=user_dict)
+    log.info('Display Intro Evals Listing')
+
+    members = ldap_get_intro_members()
+
+    ie_members = get_intro_members_without_accounts()
+
+    account_cm_count = dict([tuple(row) for row in MemberCommitteeAttendance.query.join(
+        CommitteeMeeting,
+        MemberCommitteeAttendance.meeting_id == CommitteeMeeting.id
+    ).with_entities(
+        MemberCommitteeAttendance.uid,
+        CommitteeMeeting.timestamp,
+        CommitteeMeeting.approved,
+    ).filter(
+        CommitteeMeeting.approved,
+        CommitteeMeeting.timestamp >= start_of_year()
+    ).with_entities(
+        MemberCommitteeAttendance.uid,
+        func.count(MemberCommitteeAttendance.uid) #pylint: disable=not-callable
+    ).group_by(
+        MemberCommitteeAttendance.uid
+    ).all()])
+
+    account_hm_missed = dict([tuple(row) for row in MemberHouseMeetingAttendance.query.join(
+        HouseMeeting,
+        MemberHouseMeetingAttendance.meeting_id == HouseMeeting.id
+    ).filter(
+        HouseMeeting.date >= start_of_year(),
+        MemberHouseMeetingAttendance.attendance_status == 'Absent'
+    ).with_entities(
+        MemberHouseMeetingAttendance.uid,
+            func.count(MemberHouseMeetingAttendance.uid) #pylint: disable=not-callable
+    ).group_by(
+        MemberHouseMeetingAttendance.uid
+    ).all()])
+
+    account_ts_attendance_query = MemberSeminarAttendance.query.join(
+        TechnicalSeminar,
+        MemberSeminarAttendance.seminar_id == TechnicalSeminar.id
+    ).with_entities(
+        MemberSeminarAttendance.uid,
+        TechnicalSeminar.timestamp,
+        TechnicalSeminar.approved
+    ).filter(
+        TechnicalSeminar.approved,
+        TechnicalSeminar.timestamp >= start_of_year()
+    ).with_entities(
+        MemberSeminarAttendance.uid,
+        TechnicalSeminar.name
+    ).all()
+
+    account_ts_attendance_dict = {}
+
+    for row in account_ts_attendance_query:
+        if not row[0] in account_ts_attendance_dict:
+            account_ts_attendance_dict[row[0]] = []
+
+            account_ts_attendance_dict[row[0]].append(row[1])
+
     # freshmen who have accounts
     for member in members:
         uid = member.uid
@@ -107,38 +201,31 @@ def get_fid_cm_count(member_id):
         if freshman_data.freshman_eval_result != "Pending" and internal:
             continue
 
-        h_meetings = [m.meeting_id for m in get_hm(member, only_absent=True)]
+        member_missed_hms = []
+
+        if account_hm_missed.get(uid, 0) != 0:
+            member_missed_hms = MemberHouseMeetingAttendance.query.join(
+                HouseMeeting,
+                MemberHouseMeetingAttendance.meeting_id == HouseMeeting.id
+            ).filter(
+                HouseMeeting.date >= start_of_year(),
+                MemberHouseMeetingAttendance.attendance_status == 'Absent',
+                MemberHouseMeetingAttendance.uid == uid,
+            ).with_entities(
+                func.array_agg(HouseMeeting.date)
+            ).scalar()
+
+        cms_attended = account_cm_count.get(uid, 0)
+
         member_info = {
             'name': name,
             'uid': uid,
             'eval_date': freshman_data.eval_date.strftime("%Y-%m-%d"),
             'signatures_missed': freshman_data.signatures_missed,
-            'committee_meetings': len(get_cm(member)),
-            'committee_meetings_passed': len(get_cm(member)) >= 6,
-            'house_meetings_missed':
-                [
-                    {
-                        "date": m.date.strftime("%Y-%m-%d"),
-                        "reason":
-                            MemberHouseMeetingAttendance.query.filter(
-                                MemberHouseMeetingAttendance.uid == uid,
-                                MemberHouseMeetingAttendance.meeting_id == m.id).first().excuse
-                    }
-                    for m in HouseMeeting.query.filter(
-                        HouseMeeting.id.in_(h_meetings)
-                    )
-                ],
-            'technical_seminars':
-                [seminar.name for seminar in TechnicalSeminar.query.join(
-                    MemberSeminarAttendance,
-                    MemberSeminarAttendance.seminar_id == TechnicalSeminar.id
-                    ).with_entities(
-                        TechnicalSeminar.name
-                        ).filter(
-                            TechnicalSeminar.timestamp > start_of_year(),
-                            MemberSeminarAttendance.uid == member.uid,
-                            TechnicalSeminar.approved == True # pylint: disable=singleton-comparison
-                            ).all()],
+            'committee_meetings': cms_attended,
+            'committee_meetings_passed': cms_attended >= 6,
+            'house_meetings_missed': member_missed_hms,
+            'technical_seminars': account_ts_attendance_dict.get(uid, []),
             'social_events': freshman_data.social_events,
             'comments': freshman_data.other_notes,
             'ldap_account': True,