Skip to content

Commit 90b385c

Browse files
Merge remote-tracking branch 'origin/main' into LLMO-1023-atudoran
# Conflicts: # package-lock.json # package.json
2 parents 331fbac + 157f641 commit 90b385c

File tree

127 files changed

+14518
-10319
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

127 files changed

+14518
-10319
lines changed

CHANGELOG.md

Lines changed: 413 additions & 0 deletions
Large diffs are not rendered by default.

package-lock.json

Lines changed: 3205 additions & 5315 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@adobe/spacecat-audit-worker",
3-
"version": "1.230.3",
3+
"version": "1.245.1",
44
"description": "SpaceCat Audit Worker",
55
"main": "src/index.js",
66
"type": "module",
@@ -78,37 +78,37 @@
7878
"@adobe/helix-status": "10.1.5",
7979
"@adobe/helix-universal": "5.3.0",
8080
"@adobe/helix-universal-logger": "3.0.28",
81-
"@adobe/spacecat-helix-content-sdk": "1.4.26",
82-
"@adobe/spacecat-shared-ahrefs-client": "1.9.13",
83-
"@adobe/spacecat-shared-athena-client": "1.3.8",
84-
"@adobe/spacecat-shared-data-access": "2.78.0",
85-
"@adobe/spacecat-shared-google-client": "1.4.55",
86-
"@adobe/spacecat-shared-gpt-client": "1.6.9",
87-
"@adobe/spacecat-shared-html-analyzer": "1.0.5",
88-
"@adobe/spacecat-shared-http-utils": "1.18.0",
89-
"@adobe/spacecat-shared-ims-client": "1.9.2",
90-
"@adobe/spacecat-shared-rum-api-client": "2.38.9",
81+
"@adobe/spacecat-helix-content-sdk": "1.4.27",
82+
"@adobe/spacecat-shared-data-access": "2.85.1",
83+
"@adobe/spacecat-shared-ahrefs-client": "1.10.1",
84+
"@adobe/spacecat-shared-athena-client": "1.6.1",
85+
"@adobe/spacecat-shared-google-client": "1.4.57",
86+
"@adobe/spacecat-shared-gpt-client": "1.6.10",
87+
"@adobe/spacecat-shared-html-analyzer": "1.0.6",
88+
"@adobe/spacecat-shared-http-utils": "1.18.2",
89+
"@adobe/spacecat-shared-ims-client": "1.11.1",
90+
"@adobe/spacecat-shared-rum-api-client": "2.38.10",
9191
"@adobe/spacecat-shared-rum-api-client-v1": "npm:@adobe/[email protected]",
92-
"@adobe/spacecat-shared-scrape-client": "2.3.0",
93-
"@adobe/spacecat-shared-slack-client": "1.5.29",
94-
"@adobe/spacecat-shared-tier-client": "1.2.2",
92+
"@adobe/spacecat-shared-scrape-client": "2.3.2",
93+
"@adobe/spacecat-shared-slack-client": "1.5.30",
94+
"@adobe/spacecat-shared-tier-client": "1.2.5",
9595
"@adobe/spacecat-shared-utils": "https://gitpkg.vercel.app/adobe/spacecat-shared/packages/spacecat-shared-utils?LLMO-1023-atudoran",
9696
"@adobe/structured-data-validator": "1.5.0",
97-
"@aws-sdk/client-athena": "3.922.0",
98-
"@aws-sdk/client-lambda": "3.922.0",
99-
"@aws-sdk/client-s3": "3.922.0",
100-
"@aws-sdk/client-sqs": "3.922.0",
101-
"@aws-sdk/credential-provider-node": "3.922.0",
102-
"@aws-sdk/s3-request-presigner": "3.922.0",
97+
"@aws-sdk/client-athena": "3.932.0",
98+
"@aws-sdk/client-lambda": "3.932.0",
99+
"@aws-sdk/client-s3": "3.932.0",
100+
"@aws-sdk/client-sqs": "3.932.0",
101+
"@aws-sdk/credential-provider-node": "3.932.0",
102+
"@aws-sdk/s3-request-presigner": "3.932.0",
103103
"aws-xray-sdk": "3.11.0",
104104
"cheerio": "1.1.2",
105-
"css-selector-generator": "3.7.0",
106105
"date-fns": "4.1.0",
107106
"diff": "8.0.2",
108107
"exceljs": "^4.4.0",
109108
"franc-min": "6.2.0",
110109
"get-xpath": "3.3.0",
111-
"hyparquet": "1.20.1",
110+
"hyparquet": "1.20.2",
111+
"hyparquet-writer": "0.9.0",
112112
"hyphen": "1.10.6",
113113
"is-language-code": "5.1.0",
114114
"js-beautify": "^1.15.4",
@@ -120,7 +120,7 @@
120120
"zod": "4.1.12"
121121
},
122122
"devDependencies": {
123-
"@adobe/eslint-config-helix": "3.0.13",
123+
"@adobe/eslint-config-helix": "3.0.14",
124124
"@adobe/helix-deploy": "https://gitpkg.now.sh/alinarublea/helix-deploy?main",
125125
"@adobe/helix-universal": "5.3.0",
126126
"@adobe/helix-universal-devserver": "1.1.141",
@@ -129,31 +129,31 @@
129129
"@babel/core": "7.28.5",
130130
"@babel/eslint-parser": "7.28.5",
131131
"@babel/plugin-syntax-import-assertions": "7.27.1",
132-
"@eslint/config-helpers": "0.4.2",
133-
"@redocly/cli": "2.10.0",
132+
"@eslint/config-helpers": "0.5.0",
133+
"@redocly/cli": "2.11.1",
134134
"@semantic-release/changelog": "6.0.3",
135135
"@semantic-release/exec": "7.1.0",
136136
"@semantic-release/git": "10.0.1",
137-
"@semantic-release/npm": "13.1.1",
138-
"@typescript-eslint/eslint-plugin": "8.46.2",
139-
"@typescript-eslint/parser": "8.46.2",
137+
"@semantic-release/npm": "13.1.2",
138+
"@typescript-eslint/eslint-plugin": "8.46.4",
139+
"@typescript-eslint/parser": "8.46.4",
140140
"c8": "10.1.3",
141-
"chai": "6.2.0",
141+
"chai": "6.2.1",
142142
"chai-as-promised": "8.0.2",
143143
"chokidar": "4.0.3",
144144
"dotenv": "17.2.3",
145-
"eslint": "9.39.0",
145+
"eslint": "9.39.1",
146146
"esmock": "2.7.3",
147147
"husky": "9.1.7",
148-
"hyparquet-writer": "0.9.0",
148+
"hyparquet-writer": "0.9.1",
149149
"junit-report-builder": "5.1.1",
150150
"lint-staged": "16.2.6",
151-
"mocha": "11.7.4",
151+
"mocha": "11.7.5",
152152
"mocha-multi-reporters": "1.5.1",
153153
"mocha-suppress-logs": "0.6.0",
154154
"nock": "14.0.10",
155-
"nodemon": "3.1.10",
156-
"semantic-release": "25.0.1",
155+
"nodemon": "3.1.11",
156+
"semantic-release": "25.0.2",
157157
"sinon": "21.0.0",
158158
"sinon-chai": "4.0.1",
159159
"typescript": "5.9.3",

src/accessibility/utils/data-processing.js

Lines changed: 102 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import {
1414
DeleteObjectCommand,
1515
DeleteObjectsCommand,
16+
HeadObjectCommand,
1617
ListObjectsV2Command,
1718
PutObjectCommand,
1819
} from '@aws-sdk/client-s3';
@@ -489,8 +490,9 @@ export async function createReportOpportunitySuggestion(
489490
reportMarkdown,
490491
auditData,
491492
log,
493+
context,
492494
) {
493-
const suggestions = createReportOpportunitySuggestionInstance(reportMarkdown);
495+
const suggestions = createReportOpportunitySuggestionInstance(reportMarkdown, context);
494496

495497
try {
496498
const suggestion = await opportunity.addSuggestions(suggestions);
@@ -516,6 +518,7 @@ export async function createOrUpdateDeviceSpecificSuggestion(
516518
deviceType,
517519
auditData,
518520
log,
521+
context = {},
519522
) {
520523
const createSuggestionInstance = createDeviceSpecificSuggestionInstance;
521524

@@ -534,7 +537,7 @@ export async function createOrUpdateDeviceSpecificSuggestion(
534537
currentSuggestionValue,
535538
deviceType,
536539
reportMarkdown,
537-
log,
540+
context,
538541
);
539542

540543
// Update only the suggestionValue field to avoid ElectroDB timestamp conflicts
@@ -546,7 +549,7 @@ export async function createOrUpdateDeviceSpecificSuggestion(
546549
return { suggestion: existingSuggestion };
547550
} else {
548551
// Create new suggestion
549-
suggestions = createSuggestionInstance(null, deviceType, reportMarkdown, log);
552+
suggestions = createSuggestionInstance(null, deviceType, reportMarkdown, context);
550553

551554
const suggestion = await opportunity.addSuggestions(suggestions);
552555

@@ -834,6 +837,7 @@ export async function generateReportOpportunity(
834837
deviceType.toLowerCase(),
835838
auditData,
836839
log,
840+
context,
837841
);
838842
} catch (error) {
839843
log.error(`[A11yProcessingError] Failed to create/update device-specific suggestion for ${reportName}`, error.message);
@@ -973,25 +977,98 @@ export async function sendRunImportMessage(
973977
});
974978
}
975979

980+
/**
981+
* Retrieves code path information saved in S3 bucket
982+
* @param {Object} site - The site object
983+
* @param {string} opportunityType - Opportunity type for logging
984+
* @param {Object} context - The context object containing log, s3Client, env
985+
* @returns {Promise<Object|null>} Object containing codeBucket and codePath, or null if should skip
986+
*/
987+
async function getCodeInfo(site, opportunityType, context) {
988+
const { log, s3Client, env } = context;
989+
const siteId = site.getId();
990+
const deliveryType = site.getDeliveryType();
991+
const codeConfig = site.getCode();
992+
993+
// For aem_edge delivery type, proceed without codeConfig
994+
if (!codeConfig) {
995+
if (deliveryType === 'aem_edge') {
996+
return {
997+
codeBucket: env.S3_IMPORTER_BUCKET_NAME,
998+
codePath: '',
999+
};
1000+
}
1001+
log.warn(`[${opportunityType}] [Site Id: ${siteId}] No code configuration found for site`);
1002+
return null;
1003+
}
1004+
1005+
const {
1006+
type: source, owner, repo, ref,
1007+
} = codeConfig;
1008+
1009+
const codeBucket = env.S3_IMPORTER_BUCKET_NAME;
1010+
const codePath = `code/${siteId}/${source}/${owner}/${repo}/${ref}/repository.zip`;
1011+
1012+
// Verify if the file exists in S3 bucket
1013+
let fileExists = false;
1014+
try {
1015+
await s3Client.send(new HeadObjectCommand({
1016+
Bucket: codeBucket,
1017+
Key: codePath,
1018+
}));
1019+
fileExists = true;
1020+
log.info(`[${opportunityType}] [Site Id: ${siteId}] Code file verified in S3 bucket`);
1021+
} catch (error) {
1022+
if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
1023+
log.warn(`[${opportunityType}] [Site Id: ${siteId}] Code file not found in S3: ${codePath}`);
1024+
} else {
1025+
log.error(`[${opportunityType}] [Site Id: ${siteId}] Error checking S3 file: ${error.message}`);
1026+
}
1027+
}
1028+
1029+
// Handle based on file existence and delivery type
1030+
if (!fileExists && deliveryType !== 'aem_edge') {
1031+
return null;
1032+
}
1033+
1034+
return {
1035+
codeBucket,
1036+
codePath: (!fileExists && deliveryType === 'aem_edge') ? '' : codePath,
1037+
};
1038+
}
1039+
9761040
/**
9771041
* Groups suggestions by URL, source, and issue type, then sends messages
978-
* to the importer worker for code-fix generation
1042+
* directly to Mystique for code-fix generation.
1043+
* Verifies the code file exists in S3 bucket before sending messages.
1044+
* For aem_edge delivery type, sends message with empty codePath if file doesn't exist.
1045+
* For other delivery types, skips sending if file doesn't exist.
9791046
*
9801047
* @param {Object} opportunity - The opportunity object containing suggestions
9811048
* @param {string} auditId - The audit ID
982-
* @param {Object} context - The context object containing log, sqs, env, and site
1049+
* @param {Object} site - The site object
1050+
* @param {Object} context - The context object containing log, sqs, env, s3Client, and site
9831051
* @returns {Promise<void>}
9841052
*/
985-
export async function sendCodeFixMessagesToImporter(opportunity, auditId, context) {
1053+
export async function sendCodeFixMessagesToMystique(opportunity, auditId, site, context) {
9861054
const {
987-
log, sqs, env, site,
1055+
log, sqs, env,
9881056
} = context;
9891057

9901058
const siteId = opportunity.getSiteId();
9911059
const baseUrl = site.getBaseURL();
9921060
const opportunityType = opportunity.getType();
9931061

9941062
try {
1063+
// Verify and get code path information
1064+
const codeInfo = await getCodeInfo(site, opportunityType, context);
1065+
1066+
if (!codeInfo) {
1067+
return;
1068+
}
1069+
1070+
const { codeBucket, codePath } = codeInfo;
1071+
9951072
// Get all suggestions from the opportunity
9961073
const suggestions = await opportunity.getSuggestions();
9971074
if (!suggestions || suggestions.length === 0) {
@@ -1004,16 +1081,16 @@ export async function sendCodeFixMessagesToImporter(opportunity, auditId, contex
10041081

10051082
suggestions.forEach((suggestion) => {
10061083
const suggestionData = suggestion.getData();
1007-
const { url, source = 'default', issues } = suggestionData;
1084+
const { url, source: formSource = 'default', issues } = suggestionData;
10081085

10091086
// By design, data.issues will always have length 1
10101087
if (issues && issues.length > 0) {
10111088
const issueType = issues[0].type;
1012-
const groupKey = `${url}|${source}|${issueType}`;
1089+
const groupKey = `${url}|${formSource}|${issueType}`;
10131090
if (!groupedSuggestions.has(groupKey)) {
10141091
groupedSuggestions.set(groupKey, {
10151092
url,
1016-
source,
1093+
source: formSource,
10171094
issueType,
10181095
suggestionIds: [],
10191096
});
@@ -1028,33 +1105,32 @@ export async function sendCodeFixMessagesToImporter(opportunity, auditId, contex
10281105

10291106
const messagePromises = Array.from(groupedSuggestions.values()).map(async (group) => {
10301107
const message = {
1031-
type: 'code',
1108+
type: `codefix:${opportunityType}`,
10321109
siteId,
1033-
forward: {
1034-
queue: env.QUEUE_SPACECAT_TO_MYSTIQUE,
1035-
type: `codefix:${opportunityType}`,
1036-
siteId,
1037-
auditId,
1038-
url: baseUrl,
1039-
deliveryType: site.getDeliveryType(),
1040-
data: {
1041-
opportunityId: opportunity.getId(),
1042-
suggestionIds: group.suggestionIds,
1043-
},
1110+
auditId,
1111+
url: baseUrl,
1112+
deliveryType: site.getDeliveryType(),
1113+
source: 'spacecat',
1114+
observation: 'Auto optimize form accessibility',
1115+
data: {
1116+
opportunityId: opportunity.getId(),
1117+
suggestionIds: group.suggestionIds,
1118+
codeBucket,
1119+
codePath,
10441120
},
10451121
};
10461122

10471123
try {
1048-
await sqs.sendMessage(env.IMPORT_WORKER_QUEUE_URL, message);
1049-
log.info(`[${opportunityType}] [Site Id: ${siteId}] Sent code-fix message to importer for URL: ${group.url}, source: ${group.source}, issueType: ${group.issueType}, suggestions: ${group.suggestionIds.length}`);
1124+
await sqs.sendMessage(env.QUEUE_SPACECAT_TO_MYSTIQUE, message);
1125+
log.info(`[${opportunityType}] [Site Id: ${siteId}] Sent code-fix message to Mystique for URL: ${group.url}, source: ${group.source}, issueType: ${group.issueType}, suggestions: ${group.suggestionIds.length}`);
10501126
} catch (error) {
10511127
log.error(`[${opportunityType}] [Site Id: ${siteId}] Failed to send code-fix message for URL: ${group.url}, error: ${error.message}`);
10521128
}
10531129
});
10541130

10551131
await Promise.all(messagePromises);
1056-
log.info(`[${opportunityType}] [Site Id: ${siteId}] Completed sending ${messagePromises.length} code-fix messages to importer`);
1132+
log.info(`[${opportunityType}] [Site Id: ${siteId}] Completed sending ${messagePromises.length} code-fix messages to Mystique`);
10571133
} catch (error) {
1058-
log.error(`[${opportunityType}] [Site Id: ${siteId}] Error in sendCodeFixMessagesToImporter: ${error.message}`);
1134+
log.error(`[${opportunityType}] [Site Id: ${siteId}] Error in sendCodeFixMessagesToMystique: ${error.message}`);
10591135
}
10601136
}

src/accessibility/utils/report-oppty.js

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
* governing permissions and limitations under the License.
1111
*/
1212

13+
import { Suggestion as SuggestionDataAccess } from '@adobe/spacecat-shared-data-access';
14+
1315
export function createInDepthReportOpportunity(week, year, deviceType = 'Desktop') {
1416
const capitalizedDevice = deviceType.charAt(0).toUpperCase() + deviceType.slice(1);
1517
return {
@@ -70,12 +72,13 @@ export function createBaseReportOpportunity(week, year, deviceType = 'Desktop')
7072
};
7173
}
7274

73-
export function createReportOpportunitySuggestionInstance(suggestionValue) {
75+
export function createReportOpportunitySuggestionInstance(suggestionValue, context) {
7476
return [
7577
{
7678
type: 'CODE_CHANGE',
7779
rank: 1,
78-
status: 'NEW',
80+
status: context?.site?.requiresValidation ? SuggestionDataAccess.STATUSES.PENDING_VALIDATION
81+
: SuggestionDataAccess.STATUSES.NEW,
7982
data: {
8083
suggestionValue,
8184
},
@@ -95,7 +98,7 @@ export function createOrUpdateDeviceSpecificSuggestion(
9598
suggestionValue,
9699
deviceType,
97100
markdownContent,
98-
101+
context,
99102
) {
100103
let updatedSuggestionValue;
101104

@@ -117,7 +120,7 @@ export function createOrUpdateDeviceSpecificSuggestion(
117120
updatedSuggestionValue[`accessibility-${deviceType}`] = markdownContent;
118121
}
119122

120-
return createReportOpportunitySuggestionInstance(updatedSuggestionValue);
123+
return createReportOpportunitySuggestionInstance(updatedSuggestionValue, context);
121124
}
122125

123126
export function createAccessibilityAssistiveOpportunity() {

0 commit comments

Comments
 (0)