OOM fixes (via pagination, PR-review coalescing)

src/client.ts

@@ -46,6 +46,7 @@ export class Client extends MyOctokit {
       auth: cfg.auth.oAuthToken,
       retry: {
         enabled: process.env["NODE_ENV"] !== "test",
+        retries: 1,
       },
       throttle: {
         enabled: process.env["NODE_ENV"] !== "test",

src/commands/claim.ts

@@ -216,17 +216,23 @@ async function validate(
   repoOwner: string,
   repoName: string,
 ) {
-  const issues = await this.paginate(this.issues.list, {
+  const limit = this.cfg.issues.commands.assign.newContributors.restricted;
+  let assignedCount = 0;
+
+  for await (const response of this.paginate.iterator(this.issues.list, {
     filter: "all",
     labels: this.cfg.activity.issues.inProgress,
-  });
+  })) {
+    for (const issue of response.data) {
+      if (issue.assignees?.find((assignee) => assignee.login === commenter)) {
+        assignedCount++;
+      }
+    }
 
-  const limit = this.cfg.issues.commands.assign.newContributors.restricted;
-  const assigned = issues.filter((issue) =>
-    issue.assignees?.find((assignee) => assignee.login === commenter),
-  );
+    if (assignedCount >= limit) break;
+  }
 
-  if (assigned.length >= limit) {
+  if (assignedCount >= limit) {
     const template = this.templates.get("claimRestriction");
     assertDefined(template);
     const comment = template.format({

src/events/activity.ts

@@ -5,31 +5,31 @@ import type { Client } from "../client.ts";
 import Search from "../structures/reference-search.ts";
 
 export const run = async function (this: Client) {
-  // Create array with PRs from all active repositories
+  const referenceList = new Map<string, number>();
   const repos = this.cfg.activity.check.repositories;
-  const pages = repos.map(async (repo) => {
+
+  // Process each repository sequentially to limit memory usage
+  for (const repo of repos) {
     const [repoOwner, repoName] = repo.split("/");
     assertDefined(repoOwner);
     assertDefined(repoName);
-    return this.paginate(this.pulls.list, {
+
+    for await (const response of this.paginate.iterator(this.pulls.list, {
       owner: repoOwner,
       repo: repoName,
-    });
-  });
-
-  const array = await Promise.all(pages);
-
-  // Flatten arrays of arrays with PR data
-  const pulls = array.flat();
+    })) {
+      await scrapePulls.call(this, response.data, referenceList);
+    }
+  }
 
-  await scrapePulls.call(this, pulls);
+  await scrapeInactiveIssues.call(this, referenceList);
 };
 
 async function scrapePulls(
   this: Client,
   pulls: Array<components["schemas"]["pull-request-simple"]>,
+  referenceList: Map<string, number>,
 ) {
-  const referenceList = new Map<string, number>();
   const ims = (this.cfg.activity.check.reminder ?? 0) * 86400000;
 
   for (const pull of pulls) {
@@ -73,13 +73,6 @@ async function scrapePulls(
       }
     }
   }
-
-  const issues = await this.paginate(this.issues.list, {
-    filter: "all",
-    labels: this.cfg.activity.issues.inProgress ?? undefined,
-  });
-
-  await scrapeInactiveIssues.call(this, referenceList, issues);
 }
 
 async function checkInactivePull(
@@ -118,97 +111,101 @@ async function checkInactivePull(
 async function scrapeInactiveIssues(
   this: Client,
   references: Map<string, number>,
-  issues: Array<components["schemas"]["issue"]>,
 ) {
   const ms = (this.cfg.activity.check.limit ?? 0) * 86400000;
   const ims = (this.cfg.activity.check.reminder ?? 0) * 86400000;
 
-  for (const issue of issues) {
-    const hasInactiveLabel = issue.labels.some(
-      (label) =>
-        (typeof label === "string" ? label : label.name) ===
-        this.cfg.activity.inactive,
-    );
-    if (hasInactiveLabel) continue;
-
-    let time = Date.parse(issue.updated_at);
-    const number = issue.number;
-    assertDefined(issue.repository);
-    const repoName = issue.repository.name;
-    const repoOwner = issue.repository.owner.login;
-    const issueTag = `${repoName}/${number}`;
-    const repoTag = issue.repository.full_name;
-
-    const reference = references.get(issueTag);
-    if (reference !== undefined && time < reference) time = reference;
-
-    const active = this.cfg.activity.check.repositories.includes(repoTag);
-
-    if (time + ms >= Date.now() || !active) continue;
-
-    if (
-      issue.assignees === undefined ||
-      issue.assignees === null ||
-      issue.assignees.length === 0
-    ) {
-      const comment = "**ERROR:** This active issue has no assignee.";
-      await this.issues.createComment({
-        owner: repoOwner,
-        repo: repoName,
-        issue_number: number,
-        body: comment,
-      });
-      return;
-    }
-
-    const logins = issue.assignees.map((assignee) => assignee.login);
-
-    const template = this.templates.get("inactiveWarning");
-    assertDefined(template);
+  for await (const response of this.paginate.iterator(this.issues.list, {
+    filter: "all",
+    labels: this.cfg.activity.issues.inProgress ?? undefined,
+  })) {
+    for (const issue of response.data) {
+      const hasInactiveLabel = issue.labels.some(
+        (label) =>
+          (typeof label === "string" ? label : label.name) ===
+          this.cfg.activity.inactive,
+      );
+      if (hasInactiveLabel) continue;
+
+      let time = Date.parse(issue.updated_at);
+      const number = issue.number;
+      assertDefined(issue.repository);
+      const repoName = issue.repository.name;
+      const repoOwner = issue.repository.owner.login;
+      const issueTag = `${repoName}/${number}`;
+      const repoTag = issue.repository.full_name;
+
+      const reference = references.get(issueTag);
+      if (reference !== undefined && time < reference) time = reference;
+
+      const active = this.cfg.activity.check.repositories.includes(repoTag);
+
+      if (time + ms >= Date.now() || !active) continue;
+
+      if (
+        issue.assignees === undefined ||
+        issue.assignees === null ||
+        issue.assignees.length === 0
+      ) {
+        const comment = "**ERROR:** This active issue has no assignee.";
+        await this.issues.createComment({
+          owner: repoOwner,
+          repo: repoName,
+          issue_number: number,
+          body: comment,
+        });
+        continue;
+      }
 
-    const comment = template.format({
-      assignee: logins.join(", @"),
-      remind: this.cfg.activity.check.reminder,
-      abandon: this.cfg.activity.check.limit,
-      username: this.cfg.auth.username,
-    });
+      const logins = issue.assignees.map((assignee) => assignee.login);
 
-    const comments = await template.getComments({
-      owner: repoOwner,
-      repo: repoName,
-      issue_number: number,
-    });
-
-    if (comments[0] !== undefined) {
-      await this.issues.removeAssignees({
-        owner: repoOwner,
-        repo: repoName,
-        issue_number: number,
-        assignees: logins,
-      });
-
-      const template = this.templates.get("abandonWarning");
+      const template = this.templates.get("inactiveWarning");
       assertDefined(template);
-      const warning = template.format({
+
+      const comment = template.format({
         assignee: logins.join(", @"),
-        total: (ms + ims) / 86400000,
+        remind: this.cfg.activity.check.reminder,
+        abandon: this.cfg.activity.check.limit,
         username: this.cfg.auth.username,
       });
 
-      const id = comments[0].id;
-      await this.issues.updateComment({
-        owner: repoOwner,
-        repo: repoName,
-        comment_id: id,
-        body: warning,
-      });
-    } else if (time + ims <= Date.now()) {
-      await this.issues.createComment({
+      const comments = await template.getComments({
         owner: repoOwner,
         repo: repoName,
         issue_number: number,
-        body: comment,
       });
+
+      if (comments[0] !== undefined) {
+        await this.issues.removeAssignees({
+          owner: repoOwner,
+          repo: repoName,
+          issue_number: number,
+          assignees: logins,
+        });
+
+        const template = this.templates.get("abandonWarning");
+        assertDefined(template);
+        const warning = template.format({
+          assignee: logins.join(", @"),
+          total: (ms + ims) / 86400000,
+          username: this.cfg.auth.username,
+        });
+
+        const id = comments[0].id;
+        await this.issues.updateComment({
+          owner: repoOwner,
+          repo: repoName,
+          comment_id: id,
+          body: warning,
+        });
+      } else if (time + ims <= Date.now()) {
+        await this.issues.createComment({
+          owner: repoOwner,
+          repo: repoName,
+          issue_number: number,
+          body: comment,
+        });
+      }
     }
   }
 }

src/events/responses/merge-conflict.ts

@@ -3,23 +3,39 @@ import { assertDefined, assertPresent } from "ts-extras";
 
 import type { Client } from "../../client.ts";
 
+let sweepInProgress = false;
+let sweepRequested = false;
+
 export const run = async function (
   this: Client,
   repo:
     | components["schemas"]["repository"]
     | components["schemas"]["webhook-push"]["repository"],
 ) {
-  const repoName = repo.name;
-  assertPresent(repo.owner);
-  const repoOwner = repo.owner.login;
+  if (sweepInProgress) {
+    sweepRequested = true;
+    return;
+  }
 
-  const pulls = await this.paginate(this.pulls.list, {
-    owner: repoOwner,
-    repo: repoName,
-  });
+  sweepInProgress = true;
+  try {
+    do {
+      sweepRequested = false;
+      const repoName = repo.name;
+      assertPresent(repo.owner);
+      const repoOwner = repo.owner.login;
 
-  for (const pull of pulls) {
-    await check.call(this, pull.number, repo);
+      for await (const response of this.paginate.iterator(this.pulls.list, {
+        owner: repoOwner,
+        repo: repoName,
+      })) {
+        for (const pull of response.data) {
+          await check.call(this, pull.number, repo);
+        }
+      }
+    } while (sweepRequested);
+  } finally {
+    sweepInProgress = false;
   }
 };
 
@@ -61,12 +77,18 @@ async function check(
 
   // Use a strict false check; unknown merge conflict statuses return null
   if (mergeable === false) {
-    const commits = await this.paginate(this.pulls.listCommits, {
-      owner: repoOwner,
-      repo: repoName,
-      pull_number: number,
-    });
-    const lastCommitTime = commits.at(-1)?.commit.committer?.date;
+    let lastCommitTime: string | undefined;
+    for await (const response of this.paginate.iterator(
+      this.pulls.listCommits,
+      {
+        owner: repoOwner,
+        repo: repoName,
+        pull_number: number,
+      },
+    )) {
+      const last = response.data.at(-1);
+      if (last) lastCommitTime = last.commit.committer?.date ?? lastCommitTime;
+    }
 
     const warnComment = warnings.find(
       (c) =>