Skip to content

Commit 4e4f2ba

Browse files
0x40070x4007
committed
feat: calculate quality score seems to work
1 parent 0e8607b commit 4e4f2ba

File tree

5 files changed

+189
-166
lines changed

5 files changed

+189
-166
lines changed

‎.github/workflows/short-files.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313

1414
- name: Check file length
1515
run: |
16-
IGNORE=("src/generatedFile1.ts" "src/generatedFile2.ts") # Add more files to ignore as needed
16+
IGNORE=("src/adapters/supabase/types/database.ts" "src/generatedFile2.ts") # Add more files to ignore as needed
1717
find src -name "*.ts" -type f -exec bash -c '
1818
for ignore in "${IGNORE[@]}"; do
1919
if [[ "$1" == "$ignore" ]]; then

‎src/adapters/supabase/helpers/tables/logs.ts

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
/* eslint-disable @typescript-eslint/no-explicit-any */
22
// This is disabled because logs should be able to log any type of data
33
// Normally this is forbidden
4+
// TODO: break this apart into smaller files.
45

56
import { SupabaseClient } from "@supabase/supabase-js";
67
import Runtime from "../../../../bindings/bot-runtime";

‎src/handlers/comment/handlers/issue/calculate-quality-score.ts

+45-23
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,26 @@
11
import { Comment, Issue } from "../../../../types/payload";
2-
// import OpenAI from "openai";
2+
import OpenAI from "openai";
33
import { encodingForModel } from "js-tiktoken";
44

5+
// TODO: make a filter to scrub out block quotes
56
const botCommandsAndCommentsFilter = (comment) => !comment.body.startsWith("/") && comment.user.type === "User";
67

7-
export function calculateQualScore(issue: Issue, comments: Comment[]) {
8-
const tokens = countTokensOfConversation(issue, comments);
9-
const estimatedOptimalModel = estimateOptimalModel(tokens);
10-
return { tokens, estimatedOptimalModel };
8+
export async function calculateQualScore(issue: Issue, allComments: Comment[]) {
9+
const contributorComments = allComments.filter(botCommandsAndCommentsFilter);
10+
11+
const sumOfConversationTokens = countTokensOfConversation(issue, contributorComments);
12+
const estimatedOptimalModel = estimateOptimalModel(sumOfConversationTokens);
13+
14+
const relevance = await gptRelevance(
15+
estimatedOptimalModel,
16+
issue.body,
17+
contributorComments.map((comment) => comment.body)
18+
);
19+
20+
if (relevance.length != contributorComments.length) {
21+
throw new Error("Relevance scores returned from OpenAI do not match the number of comments");
22+
}
23+
return { tokens: sumOfConversationTokens, estimatedOptimalModel };
1124
}
1225

1326
function estimateOptimalModel(sumOfTokens: number) {
@@ -26,40 +39,49 @@ function estimateOptimalModel(sumOfTokens: number) {
2639
}
2740

2841
function countTokensOfConversation(issue: Issue, comments: Comment[]) {
29-
const gpt3TurboEncoder = encodingForModel("gpt-3.5-turbo");
3042
const specificationComment = issue.body;
3143
if (!specificationComment) {
3244
throw new Error("Issue specification comment is missing");
3345
}
3446

35-
const contributorCommentsWithTokens = comments.filter(botCommandsAndCommentsFilter).map((comment) => {
47+
const gpt3TurboEncoder = encodingForModel("gpt-3.5-turbo");
48+
const contributorCommentsWithTokens = comments.map((comment) => {
3649
return {
3750
tokens: gpt3TurboEncoder.encode(comment.body),
3851
comment,
3952
};
4053
});
4154

42-
// const contributorCommentsTokens = contributorCommentsWithTokens.map(({ tokens }) => tokens);
4355
const sumOfContributorTokens = contributorCommentsWithTokens.reduce((acc, { tokens }) => acc + tokens.length, 0);
4456
const specificationTokens = gpt3TurboEncoder.encode(specificationComment);
4557
const sumOfSpecificationTokens = specificationTokens.length;
4658
const totalSumOfTokens = sumOfSpecificationTokens + sumOfContributorTokens;
47-
// const estimatedOptimalModel = estimateOptimalModel(totalSumOfTokens);
4859

49-
// const buffer = {
50-
// totalSumOfTokens,
51-
// estimatedOptimalModel,
52-
// };
53-
// return buffer;
5460
return totalSumOfTokens;
5561
}
5662

57-
// async function gpt() {
58-
// // /v1/chat/completions
59-
// const openai = new OpenAI(); // apiKey: // defaults to process.env["OPENAI_API_KEY"]
60-
// const params: OpenAI.Chat.ChatCompletionCreateParams = {
61-
// messages: [{ role: "user", content: "Say this is a test" }],
62-
// model: "gpt-3.5-turbo",
63-
// };
64-
// const chatCompletion: OpenAI.Chat.ChatCompletion = await openai.chat.completions.create(params);
65-
// }
63+
async function gptRelevance(model: string, ISSUE_SPECIFICATION_BODY: string, CONVERSATION_STRINGS: string[]) {
64+
const openai = new OpenAI(); // apiKey: // defaults to process.env["OPENAI_API_KEY"]
65+
const PROMPT = `I need to evaluate the relevance of GitHub contributors' comments to a specific issue specification. Specifically, I'm interested in how much each comment helps to further define the issue specification or contributes new information or research relevant to the issue. Please provide a float between 0 and 1 to represent the degree of relevance. A score of 1 indicates that the comment is entirely relevant and adds significant value to the issue, whereas a score of 0 indicates no relevance or added value. Each contributor's comment is on a new line.\n\nIssue Specification:\n\`\`\`\n${ISSUE_SPECIFICATION_BODY}\n\`\`\`\n\nConversation:\n\`\`\`\n${CONVERSATION_STRINGS.join(
66+
"\n"
67+
)}\n\`\`\`\n\n\nTo what degree are each of the comments in the conversation relevant and valuable to further defining the issue specification? Please reply with an array of float numbers between 0 and 1, corresponding to each comment in the order they appear. Each float should represent the degree of relevance and added value of the comment to the issue.`;
68+
console.trace({ PROMPT });
69+
const response: OpenAI.Chat.ChatCompletion = await openai.chat.completions.create({
70+
model: model,
71+
messages: [
72+
{
73+
role: "system",
74+
content: PROMPT,
75+
},
76+
],
77+
temperature: 1,
78+
max_tokens: 1024,
79+
top_p: 1,
80+
frequency_penalty: 0,
81+
presence_penalty: 0,
82+
});
83+
84+
const parsedResponse = JSON.parse(response.choices[0].message.content as "[1, 1, 0, 0]");
85+
console.trace({ parsedResponse });
86+
return parsedResponse;
87+
}

‎src/handlers/comment/handlers/issue/calculate-typings.ts

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// import { User } from "../../../../types";
2+
3+
// type ContributionLocation = "issue" | "review";
4+
// type ContributionStyle = "issuer" | "assignee" | "collaborator" | "default";
5+
// type Role =
6+
// | "issueIssuer"
7+
// | "issueAssignee"
8+
// | "issueCollaborator"
9+
// | "issueDefault"
10+
// | "reviewIssuer"
11+
// | "reviewAssignee"
12+
// | "reviewCollaborator"
13+
// | "reviewDefault";
14+
15+
// type DevPoolContributor = {
16+
// contribution: {
17+
// role: Role;
18+
// style: ContributionStyle;
19+
// };
20+
// records: {
21+
// comments: [
22+
// {
23+
// location: ContributionLocation;
24+
// issueId: number;
25+
// commentId: number;
26+
// body: string;
27+
// score: {
28+
// quantitative: number;
29+
// qualitative: string;
30+
// };
31+
// }
32+
// ];
33+
// review: [];
34+
// };
35+
// user: User;
36+
// walletAddress: string;
37+
// };
38+
39+
// type Payments = {
40+
// contributors: User[];
41+
// };
42+
43+
//
44+
45+
type ContributionLocation = "issue" | "review";
46+
type ContributionStyle = "issuer" | "assignee" | "collaborator" | "default";
47+
type Role =
48+
| "issueIssuer"
49+
| "issueAssignee"
50+
| "issueCollaborator"
51+
| "issueDefault"
52+
| "reviewIssuer"
53+
| "reviewAssignee"
54+
| "reviewCollaborator"
55+
| "reviewDefault";
56+
57+
type ReviewState = "commented" | "approved" | "requestChanges" | "dismissed";
58+
59+
type CommentScoringConfig = {
60+
// wordCredit: number; // credit per word
61+
// listItemCredit: number; // credit per list item
62+
// imageCredit: number; // credit per image
63+
// linkCredit: number; // credit per link
64+
// codeBlockCredit: number; // credit per code block
65+
};
66+
67+
export type CommentScore = {
68+
qualitative: number; // a float between 0 and 1
69+
quantitative: number; // calculated based on CommentScoringConfig
70+
finalScore: number; // qualitative * quantitative
71+
};
72+
73+
type LabelAction = {
74+
label: string;
75+
added: boolean; // true if added, false if removed
76+
};
77+
78+
type PaymentConfig = {
79+
// Define how much each role and action is worth in monetary terms
80+
[key in Role]: {
81+
comment: number;
82+
// labelPriority: number;
83+
// labelTime: number;
84+
// codeCommit: number;
85+
// edit: number;
86+
reviewState: {
87+
[key in ReviewState]: number;
88+
};
89+
// timeSpent: number; // Per unit time
90+
};
91+
};
92+
93+
type ContributionRecord = {
94+
comments: {
95+
location: ContributionLocation;
96+
issueId: number;
97+
commentId: number;
98+
body: string;
99+
timestamp: string;
100+
score: CommentScore;
101+
}[];
102+
// labels: {
103+
// issueId: number;
104+
// actions: LabelAction[];
105+
// timestamp: string;
106+
// }[];
107+
// commits: {
108+
// pullRequestId: number;
109+
// commitId: string;
110+
// timestamp: string;
111+
// }[];
112+
// edits: {
113+
// location: ContributionLocation;
114+
// issueId: number;
115+
// editedField: "description" | "comment"; // Add more fields if necessary
116+
// timestamp: string;
117+
// }[];
118+
reviewStates: {
119+
pullRequestId: number;
120+
state: ReviewState;
121+
timestamp: string;
122+
}[];
123+
// timeSpent: number; // In some unit, e.g., minutes
124+
};
125+
126+
type DevPoolContributor = {
127+
contribution: {
128+
role: Role;
129+
style: ContributionStyle;
130+
};
131+
records: ContributionRecord;
132+
user: User;
133+
walletAddress: string;
134+
};
135+
136+
type Payments = {
137+
contributors: DevPoolContributor[];
138+
totalPayment: number;
139+
};
140+
141+
// Your existing logic here

0 commit comments

Comments
 (0)