diff --git a/src/Common/Entities/Activity.ts b/src/Common/Entities/Activity.ts index 585f6a2a..194c47cf 100644 --- a/src/Common/Entities/Activity.ts +++ b/src/Common/Entities/Activity.ts @@ -1,4 +1,4 @@ -import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index} from "typeorm"; +import {Entity, Column, ManyToOne, PrimaryColumn, OneToMany, Index, DataSource, JoinColumn} from "typeorm"; import {AuthorEntity} from "./AuthorEntity"; import {Subreddit} from "./Subreddit"; import {CMEvent} from "./CMEvent"; @@ -6,6 +6,8 @@ import {asComment, getActivityAuthorName, parseRedditFullname, redditThingTypeTo import {activityReports, ActivityType, Report, SnoowrapActivity} from "../Infrastructure/Reddit"; import {ActivityReport} from "./ActivityReport"; import dayjs, {Dayjs} from "dayjs"; +import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; +import {Comment, Submission} from 'snoowrap/dist/objects'; export interface ActivityEntityOptions { id: string @@ -45,7 +47,7 @@ export class Activity { @Column({name: 'name'}) name!: string; - @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert']}) + @ManyToOne(type => Subreddit, sub => sub.activities, {cascade: ['insert'], eager: true}) subreddit!: Subreddit; @Column("varchar", {length: 20}) @@ -58,17 +60,18 @@ export class Activity { @Column("text") permalink!: string; - @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert']}) + @ManyToOne(type => AuthorEntity, author => author.activities, {cascade: ['insert'], eager: true}) author!: AuthorEntity; - @OneToMany(type => CMEvent, act => act.activity) // note: we will create author property in the Photo class below + @OneToMany(type => CMEvent, act => act.activity) actionedEvents!: CMEvent[] - @ManyToOne(type => Activity, obj => obj.comments, {nullable: true}) + @ManyToOne('Activity', 'comments', {nullable: true, cascade: ['insert']}) + @JoinColumn({name: 'submission_id'}) submission?: Activity; - @OneToMany(type => Activity, obj => obj.submission, {nullable: true}) - comments!: Activity[]; + @OneToMany('Activity', 'submission', {nullable: true}) + comments?: Activity[]; @OneToMany(type => ActivityReport, act => act.activity, {cascade: ['insert'], eager: true}) reports: ActivityReport[] | undefined @@ -151,10 +154,12 @@ export class Activity { return false; } - static fromSnoowrapActivity(subreddit: Subreddit, activity: SnoowrapActivity, lastKnownStateTimestamp?: dayjs.Dayjs | undefined) { + static async fromSnoowrapActivity(activity: SnoowrapActivity, options: fromSnoowrapOptions | undefined = {}) { + let submission: Activity | undefined; let type: ActivityType = 'submission'; let content: string; + const subreddit = await Subreddit.fromSnoowrap(activity.subreddit, options?.db); if(asComment(activity)) { type = 'comment'; content = activity.body; @@ -179,8 +184,30 @@ export class Activity { submission }); - entity.syncReports(activity, lastKnownStateTimestamp); + entity.syncReports(activity, options.lastKnownStateTimestamp); return entity; } + + toSnoowrap(client: ExtendedSnoowrap): SnoowrapActivity { + let act: SnoowrapActivity; + if(this.type === 'submission') { + act = new Submission({name: this.id, id: this.name}, client, false); + act.title = this.content; + } else { + act = new Comment({name: this.id, id: this.name}, client, false); + act.link_id = this.submission?.id as string; + act.body = this.content; + } + act.permalink = this.permalink; + act.subreddit = this.subreddit.toSnoowrap(client); + act.author = this.author.toSnoowrap(client); + + return act; + } +} + +export interface fromSnoowrapOptions { + lastKnownStateTimestamp?: dayjs.Dayjs | undefined + db?: DataSource } diff --git a/src/Common/Entities/AuthorEntity.ts b/src/Common/Entities/AuthorEntity.ts index 3b773972..a1b96bf1 100644 --- a/src/Common/Entities/AuthorEntity.ts +++ b/src/Common/Entities/AuthorEntity.ts @@ -1,5 +1,8 @@ import {Entity, Column, PrimaryColumn, OneToMany} from "typeorm"; import {Activity} from "./Activity"; +import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; +import {SnoowrapActivity} from "../Infrastructure/Reddit"; +import {RedditUser} from "snoowrap/dist/objects"; @Entity({name: 'Author'}) export class AuthorEntity { @@ -11,11 +14,15 @@ export class AuthorEntity { name!: string; @OneToMany(type => Activity, act => act.author) - activities!: Activity[] + activities!: Promise constructor(data?: any) { if(data !== undefined) { this.name = data.name; } } + + toSnoowrap(client: ExtendedSnoowrap): RedditUser { + return new RedditUser({name: this.name, id: this.id}, client, false); + } } diff --git a/src/Common/Entities/DispatchedEntity.ts b/src/Common/Entities/DispatchedEntity.ts index 0c61ef09..848814c7 100644 --- a/src/Common/Entities/DispatchedEntity.ts +++ b/src/Common/Entities/DispatchedEntity.ts @@ -6,7 +6,7 @@ import { ManyToOne, PrimaryColumn, BeforeInsert, - AfterLoad + AfterLoad, JoinColumn } from "typeorm"; import { ActivityDispatch @@ -22,15 +22,15 @@ import Comment from "snoowrap/dist/objects/Comment"; import {ColumnDurationTransformer} from "./Transformers"; import { RedditUser } from "snoowrap/dist/objects"; import {ActivitySourceTypes, DurationVal, NonDispatchActivitySourceValue, onExistingFoundBehavior} from "../Infrastructure/Atomic"; +import {Activity} from "./Activity"; @Entity({name: 'DispatchedAction'}) export class DispatchedEntity extends TimeAwareRandomBaseEntity { - @Column() - activityId!: string - - @Column() - author!: string + //@ManyToOne(type => Activity, obj => obj.dispatched, {cascade: ['insert'], eager: true, nullable: false}) + @ManyToOne(type => Activity, undefined, {cascade: ['insert'], eager: true, nullable: false}) + @JoinColumn({name: 'activityId'}) + activity!: Activity @Column({ type: 'int', @@ -82,11 +82,10 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity { }}) tardyTolerant!: boolean | Duration - constructor(data?: ActivityDispatch & { manager: ManagerEntity }) { + constructor(data?: HydratedActivityDispatch) { super(); if (data !== undefined) { - this.activityId = data.activity.name; - this.author = getActivityAuthorName(data.activity.author); + this.activity = data.activity; this.delay = data.delay; this.createdAt = data.queuedAt; this.type = data.type; @@ -151,20 +150,7 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity { } async toActivityDispatch(client: ExtendedSnoowrap): Promise { - const redditThing = parseRedditFullname(this.activityId); - if(redditThing === undefined) { - throw new Error(`Could not parse reddit ID from value '${this.activityId}'`); - } - let activity: Comment | Submission; - if (redditThing?.type === 'comment') { - // @ts-ignore - activity = await client.getComment(redditThing.id); - } else { - // @ts-ignore - activity = await client.getSubmission(redditThing.id); - } - activity.author = new RedditUser({name: this.author}, client, false); - activity.id = redditThing.id; + let activity = this.activity.toSnoowrap(client); return { id: this.id, queuedAt: this.createdAt, @@ -176,8 +162,13 @@ export class DispatchedEntity extends TimeAwareRandomBaseEntity { cancelIfQueued: this.cancelIfQueued, identifier: this.identifier, type: this.type, - author: this.author, + author: activity.author.name, dryRun: this.dryRun } } } + +export interface HydratedActivityDispatch extends Omit { + activity: Activity + manager: ManagerEntity +} diff --git a/src/Common/Entities/Subreddit.ts b/src/Common/Entities/Subreddit.ts index 53bbcb1f..69adeebd 100644 --- a/src/Common/Entities/Subreddit.ts +++ b/src/Common/Entities/Subreddit.ts @@ -1,5 +1,7 @@ -import {Entity, Column, PrimaryColumn, OneToMany, Index} from "typeorm"; +import {Entity, Column, PrimaryColumn, OneToMany, Index, DataSource} from "typeorm"; import {Activity} from "./Activity"; +import {ExtendedSnoowrap} from "../../Utils/SnoowrapClients"; +import {Subreddit as SnoowrapSubreddit} from "snoowrap/dist/objects"; export interface SubredditEntityOptions { id: string @@ -25,4 +27,18 @@ export class Subreddit { this.name = data.name; } } + + toSnoowrap(client: ExtendedSnoowrap): SnoowrapSubreddit { + return new SnoowrapSubreddit({display_name: this.name, name: this.id}, client, false); + } + + static async fromSnoowrap(subreddit: SnoowrapSubreddit, db?: DataSource) { + if(db !== undefined) { + const existing = await db.getRepository(Subreddit).findOneBy({name: subreddit.display_name}); + if(existing) { + return existing; + } + } + return new Subreddit({id: await subreddit.name, name: await subreddit.display_name}); + } } diff --git a/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts b/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts new file mode 100644 index 00000000..3681d2ee --- /dev/null +++ b/src/Common/Migrations/Database/Server/1667415256831-delayedReset.ts @@ -0,0 +1,19 @@ +import {MigrationInterface, QueryRunner, TableColumn} from "typeorm" + +export class delayedReset1667415256831 implements MigrationInterface { + + public async up(queryRunner: QueryRunner): Promise { + queryRunner.connection.logger.logSchemaBuild('Truncating (removing) existing Dispatched Actions due to internal structural changes'); + await queryRunner.clearTable('DispatchedAction'); + await queryRunner.changeColumn('DispatchedAction', 'author', new TableColumn({ + name: 'author', + type: 'varchar', + length: '150', + isNullable: true + })); + } + + public async down(queryRunner: QueryRunner): Promise { + } + +} diff --git a/src/Subreddit/Manager.ts b/src/Subreddit/Manager.ts index 77b6e6f6..d948a278 100644 --- a/src/Subreddit/Manager.ts +++ b/src/Subreddit/Manager.ts @@ -975,7 +975,7 @@ export class Manager extends EventEmitter implements RunningStates { let shouldPersistReports = false; if (existingEntity === null) { - activityEntity = Activity.fromSnoowrapActivity(this.managerEntity.subreddit, activity, lastKnownStateTimestamp); + activityEntity = await Activity.fromSnoowrapActivity(activity, {lastKnownStateTimestamp, db: this.resources.database}); // always persist if activity is not already persisted and any reports exist if (item.num_reports > 0) { shouldPersistReports = true; @@ -1189,7 +1189,7 @@ export class Manager extends EventEmitter implements RunningStates { // @ts-ignore const subProxy = await this.client.getSubmission((item as Comment).link_id); const sub = await this.resources.getActivity(subProxy); - subActivity = await this.activityRepo.save(Activity.fromSnoowrapActivity(this.managerEntity.subreddit, sub)); + subActivity = await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.resources.database})); } event.activity.submission = subActivity; diff --git a/src/Subreddit/SubredditResources.ts b/src/Subreddit/SubredditResources.ts index 216f99cb..954a6b7e 100644 --- a/src/Subreddit/SubredditResources.ts +++ b/src/Subreddit/SubredditResources.ts @@ -69,7 +69,16 @@ import {cacheTTLDefaults, createHistoricalDisplayDefaults,} from "../Common/defa import {ExtendedSnoowrap} from "../Utils/SnoowrapClients"; import dayjs, {Dayjs} from "dayjs"; import ImageData from "../Common/ImageData"; -import {Between, DataSource, DeleteQueryBuilder, LessThan, Repository, SelectQueryBuilder} from "typeorm"; +import { + Between, Brackets, + DataSource, + DeleteQueryBuilder, + In, + LessThan, + NotBrackets, + Repository, + SelectQueryBuilder +} from "typeorm"; import {CMEvent as ActionedEventEntity, CMEvent} from "../Common/Entities/CMEvent"; import {RuleResultEntity} from "../Common/Entities/RuleResultEntity"; import globrex from 'globrex'; @@ -162,6 +171,8 @@ import {ActivitySource} from "../Common/ActivitySource"; import {SubredditResourceOptions} from "../Common/Subreddit/SubredditResourceInterfaces"; import {SubredditStats} from "./Stats"; import {CMCache} from "../Common/Cache"; +import { Activity } from '../Common/Entities/Activity'; +import {FindOptionsWhere} from "typeorm/find-options/FindOptionsWhere"; export const DEFAULT_FOOTER = '\r\n*****\r\nThis action was performed by [a bot.]({{botLink}}) Mention a moderator or [send a modmail]({{modmailLink}}) if you have any ideas, questions, or concerns about this action.'; @@ -193,6 +204,7 @@ export class SubredditResources { botAccount?: string; dispatchedActivityRepo: Repository activitySourceRepo: Repository + activityRepo: Repository retention?: EventRetentionPolicyRange managerEntity: ManagerEntity botEntity: Bot @@ -229,6 +241,7 @@ export class SubredditResources { this.database = database; this.dispatchedActivityRepo = this.database.getRepository(DispatchedEntity); this.activitySourceRepo = this.database.getRepository(ActivitySourceEntity); + this.activityRepo = this.database.getRepository(Activity); this.retention = retention; //this.prefix = prefix; this.client = client; @@ -404,21 +417,25 @@ export class SubredditResources { } }, relations: { - manager: true + manager: true, + activity: { + submission: true + } } }); const now = dayjs(); + const toRemove = []; for(const dAct of dispatchedActivities) { const shouldDispatchAt = dAct.createdAt.add(dAct.delay.asSeconds(), 'seconds'); let tardyHint = ''; if(shouldDispatchAt.isBefore(now)) { - let tardyHint = `Activity ${dAct.activityId} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`; + let tardyHint = `Activity ${dAct.activity.id} queued at ${dAct.createdAt.format('YYYY-MM-DD HH:mm:ssZ')} for ${dAct.delay.humanize()} is now LATE`; if(dAct.tardyTolerant === true) { tardyHint += ` but was configured as ALWAYS 'tardy tolerant' so will be dispatched immediately`; } else if(dAct.tardyTolerant === false) { tardyHint += ` and was not configured as 'tardy tolerant' so will be dropped`; this.logger.warn(tardyHint); - await this.removeDelayedActivity(dAct.id); + toRemove.push(dAct.id); continue; } else { // see if its within tolerance @@ -426,7 +443,7 @@ export class SubredditResources { if(latest.isBefore(now)) { tardyHint += ` and IS NOT within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dropped`; this.logger.warn(tardyHint); - await this.removeDelayedActivity(dAct.id); + toRemove.push(dAct.id); continue; } else { tardyHint += `but is within tardy tolerance of ${dAct.tardyTolerant.humanize()} of planned dispatch time so will be dispatched immediately`; @@ -439,27 +456,115 @@ export class SubredditResources { try { this.delayedItems.push(await dAct.toActivityDispatch(this.client)) } catch (e) { - this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activityId} from database delayed activities to in-app delayed activities queue`, {cause: e})); + this.logger.warn(new ErrorWithCause(`Unable to add Activity ${dAct.activity.id} from database delayed activities to in-app delayed activities queue`, {cause: e})); } } + if(toRemove.length > 0) { + await this.removeDelayedActivity(toRemove); + } } } async addDelayedActivity(data: ActivityDispatch) { - const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity})); + // TODO merge this with getActivity or something... + if(asComment(data.activity)) { + const existingSub = await this.activityRepo.findOneBy({_id: data.activity.link_id}); + if(existingSub === null) { + const sub = await this.getActivity(new Submission({name: data.activity.link_id}, this.client, false)); + await this.activityRepo.save(await Activity.fromSnoowrapActivity(sub, {db: this.database})); + } + } + const dEntity = await this.dispatchedActivityRepo.save(new DispatchedEntity({...data, manager: this.managerEntity, activity: await Activity.fromSnoowrapActivity(data.activity, {db: this.database})})); data.id = dEntity.id; this.delayedItems.push(data); } async removeDelayedActivity(val?: string | string[]) { - if(val === undefined) { - await this.dispatchedActivityRepo.delete({manager: {id: this.managerEntity.id}}); - this.delayedItems = []; - } else { + let dispatched: DispatchedEntity[] = []; + const where: FindOptionsWhere = { + manager: { + id: this.managerEntity.id + } + }; + + if(val !== undefined) { const ids = typeof val === 'string' ? [val] : val; - await this.dispatchedActivityRepo.delete(ids); - this.delayedItems = this.delayedItems.filter(x => !ids.includes(x.id)); + where.id = In(ids); + } + + dispatched = await this.dispatchedActivityRepo.find({ + where, + relations: { + manager: true, + activity: { + actionedEvents: true, + submission: { + actionedEvents: true + } + } + } + }); + + const actualDispatchedIds = dispatched.map(x => x.id); + this.logger.debug(`${actualDispatchedIds.length} marked for deletion`, {leaf: 'Delayed Activities'}); + + // get potential activities to delete + // but only include activities that don't have any actionedEvents + let activityIdsToDelete = Array.from(dispatched.reduce((acc, curr) => { + if(curr.activity.actionedEvents === null || curr.activity.actionedEvents.length === 0) { + acc.add(curr.activity.id); + } + if(curr.activity.submission !== undefined && curr.activity.submission !== null) { + if(curr.activity.submission.actionedEvents === null || curr.activity.submission.actionedEvents.length === 0) { + acc.add(curr.activity.submission.id); + } + } + return acc; + }, new Set())); + const rawActCount = activityIdsToDelete.length; + let activeActCount = 0; + + // if we have any potential activities to delete we now need to get any dispatched actions that reference these activities + // that are NOT the ones we are going to delete + if(activityIdsToDelete.length > 0) { + const activeDispatchedQuery = this.dispatchedActivityRepo.createQueryBuilder('dis') + .leftJoinAndSelect('dis.activity', 'activity') + .leftJoinAndSelect('activity.submission', 'submission') + .where(new NotBrackets((qb) => { + qb.where('dis.id IN (:...currIds)', {currIds: actualDispatchedIds}); + })) + .andWhere(new Brackets((qb) => { + qb.where('activity._id IN (:...actMainIds)', {actMainIds: activityIdsToDelete}) + qb.orWhere('submission._id IN (:...actSubIds)', {actSubIds: activityIdsToDelete}) + })); + //const sql = activeDispatchedQuery.getSql(); + const activeDispatched = await activeDispatchedQuery.getMany(); + + // all activity ids, from the actions to delete, that are being used by dispatched actions that are NOT the ones we are going to delete + const activeDispatchedIds = Array.from(activeDispatched.reduce((acc, curr) => { + acc.add(curr.activity.id); + if(curr.activity.submission !== undefined && curr.activity.submission !== null) { + acc.add(curr.activity.submission.id); + } + return acc; + }, new Set())); + activeActCount = activeDispatchedIds.length; + + // filter out any that are still in use + activityIdsToDelete = activityIdsToDelete.filter(x => !activeDispatchedIds.includes(x)); + } + + this.logger.debug(`Marked ${activityIdsToDelete.length} Activities created, by Delayed, for deletion (${rawActCount} w/o Events | ${activeActCount} used by other Delayed Activities)`, {leaf: 'Delayed Activities'}); + + if(actualDispatchedIds.length > 0) { + await this.dispatchedActivityRepo.delete(actualDispatchedIds); + } else { + this.logger.warn('No dispatched ids found to delete'); + } + if(activityIdsToDelete.length > 0) { + await this.activityRepo.delete(activityIdsToDelete); } + this.delayedItems = this.delayedItems.filter(x => !actualDispatchedIds.includes(x.id)); } async initStats() { diff --git a/src/Web/assets/views/status.ejs b/src/Web/assets/views/status.ejs index b9aa0ae2..eb679b28 100644 --- a/src/Web/assets/views/status.ejs +++ b/src/Web/assets/views/status.ejs @@ -1297,7 +1297,7 @@ const durationDayjs = dayjs.duration(x.duration, 'seconds'); const durationDisplay = durationDayjs.humanize(); const cancelLink = `CANCEL`; - return `
A ${x.submissionId !== undefined ? 'Comment' : 'Submission'}${isAll ? ` in ${x.subreddit} ` : ''} by ${x.author} queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}
`; + return `
A ${x.submissionId !== undefined ? 'Comment' : 'Submission'} by ${x.author}${isAll ? `, dispatched in ${x.subreddit} ,` : ''} queued by ${x.source} at ${queuedAtDisplay} for ${durationDisplay} (dispatches ${durationUntilNow.humanize(true)}) -- ${cancelLink}
`; }); //let sub = resp.name; if(sub === 'All') { diff --git a/src/util.ts b/src/util.ts index 6957cc96..b5df03e4 100644 --- a/src/util.ts +++ b/src/util.ts @@ -2855,7 +2855,7 @@ export const generateSnoowrapEntityFromRedditThing = (data: RedditThing, client: case 'user': return new RedditUser({id: data.val}, client, false); case 'subreddit': - return new Subreddit({id: data.val}, client, false); + return new Subreddit({name: data.val}, client, false); case 'message': return new PrivateMessage({id: data.val}, client, false)