Skip to content

Commit 2611109

Browse files
maxachismaxachis
committed
Begin draft
1 parent f541d6d commit 2611109

File tree

211 files changed

+2583
-231
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

211 files changed

+2583
-231
lines changed

‎alembic/versions/2025_10_28_1539-a57c3b5b6e93_add_sync_log_table.py‎

Lines changed: 369 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,369 @@
1+
"""Add sync_log table
2+
3+
Revision ID: a57c3b5b6e93
4+
Revises: f32ba7664e9f
5+
Create Date: 2025-10-28 15:39:50.494489
6+
7+
"""
8+
from typing import Sequence, Union
9+
10+
from alembic import op
11+
import sqlalchemy as sa
12+
13+
from src.util.alembic_helpers import created_at_column
14+
15+
# revision identifiers, used by Alembic.
16+
revision: str = 'a57c3b5b6e93'
17+
down_revision: Union[str, None] = 'f32ba7664e9f'
18+
branch_labels: Union[str, Sequence[str], None] = None
19+
depends_on: Union[str, Sequence[str], None] = None
20+
21+
22+
def last_synced_at_column():
23+
return sa.Column(
24+
'last_synced_at',
25+
sa.DateTime(),
26+
nullable=False,
27+
server_default=sa.func.now()
28+
)
29+
30+
31+
def _add_link_table_modification_triggers():
32+
op.execute("""
33+
-- trigger func that "touches" parent rows hit by changes to the link table
34+
CREATE OR REPLACE FUNCTION touch_url_from_agency_link()
35+
RETURNS trigger
36+
LANGUAGE plpgsql AS $$
37+
BEGIN
38+
-- UNION to cover INSERT/UPDATE (NEW TABLE) and DELETE (OLD TABLE)
39+
UPDATE urls u
40+
SET updated_at = clock_timestamp() -- better than now() for long txns
41+
FROM (
42+
SELECT DISTINCT url_id FROM newtab
43+
UNION
44+
SELECT DISTINCT url_id FROM oldtab
45+
) AS hit
46+
WHERE u.id = hit.url_id;
47+
48+
RETURN NULL; -- statement-level trigger
49+
END $$;
50+
51+
-- statement-level trigger with transition tables
52+
CREATE TRIGGER trg_link_touch_parent
53+
AFTER INSERT OR UPDATE OR DELETE ON link_parent_child
54+
REFERENCING NEW TABLE AS newtab OLD TABLE AS oldtab
55+
FOR EACH STATEMENT
56+
EXECUTE FUNCTION touch_parent_from_link();
57+
58+
""")
59+
60+
op.execute(
61+
"""
62+
-- trigger func that "touches" agency rows hit by changes to the link_agencies_locations table
63+
CREATE OR REPLACE FUNCTION touch_agency_from_location_link()
64+
RETURNS trigger
65+
LANGUAGE plpgsql AS
66+
$$
67+
BEGIN
68+
-- UNION to cover INSERT/UPDATE (NEW TABLE) and DELETE (OLD TABLE)
69+
UPDATE agencies a
70+
SET updated_at = clock_timestamp() -- better than now() for long txns
71+
FROM (SELECT DISTINCT agency_id
72+
FROM newtab
73+
UNION
74+
SELECT DISTINCT agency_id
75+
FROM oldtab) AS hit
76+
WHERE a.id = hit.agency_id;
77+
78+
RETURN NULL; -- statement-level trigger
79+
END
80+
$$;
81+
82+
-- statement-level trigger with transition tables
83+
CREATE TRIGGER trg_link_touch_parent
84+
AFTER INSERT OR UPDATE OR DELETE
85+
ON link_agencies_locations
86+
REFERENCING NEW TABLE AS newtab OLD TABLE AS oldtab
87+
FOR EACH STATEMENT
88+
EXECUTE FUNCTION touch_agency_from_location_link();
89+
"""
90+
)
91+
92+
93+
def upgrade() -> None:
94+
_create_sync_log()
95+
_create_ds_agency_link()
96+
_migrate_agency_ids_to_ds_agency_link()
97+
remove_id_column_from_agencies()
98+
rename_agency_id_to_id()
99+
_rename_existing_tables_to_ds_app_format()
100+
_alter_ds_app_link_data_source_table()
101+
_alter_ds_app_link_meta_url_table()
102+
_add_flag_deletion_tables()
103+
_add_last_synced_at_columns()
104+
_add_link_table_modification_triggers()
105+
106+
def _add_last_synced_at_columns():
107+
op.add_column(
108+
'ds_app_link_data_source',
109+
last_synced_at_column()
110+
)
111+
op.add_column(
112+
'ds_app_link_meta_url',
113+
last_synced_at_column()
114+
)
115+
116+
117+
def _alter_ds_app_link_data_source_table():
118+
# Drop unique constraint for data source id
119+
op.drop_constraint(
120+
'uq_url_data_sources_data_source_id',
121+
'ds_app_link_data_source',
122+
type_='unique'
123+
)
124+
# Drop primary keys
125+
op.drop_constraint(
126+
'url_data_sources_pkey',
127+
'ds_app_link_data_source',
128+
type_='primary'
129+
)
130+
# Rename `data_source_id` to `ds_data_source_id`
131+
op.alter_column(
132+
'ds_app_link_data_source',
133+
'data_source_id',
134+
new_column_name='ds_data_source_id',
135+
)
136+
# Add new primary key
137+
op.create_primary_key(
138+
'ds_app_link_data_source_pkey',
139+
'ds_app_link_data_source',
140+
['ds_data_source_id']
141+
)
142+
143+
# Drop url_id foreign key
144+
op.drop_constraint(
145+
'url_data_sources_url_id_fkey',
146+
'ds_app_link_data_source',
147+
type_='foreignkey'
148+
)
149+
# Recreate foreign key with ON DELETE SET NULL
150+
op.create_foreign_key(
151+
'ds_app_link_data_source_url_id_fkey',
152+
'ds_app_link_data_source',
153+
'urls',
154+
['url_id'],
155+
['id'],
156+
ondelete='SET NULL'
157+
)
158+
# Alter url_id column to be nullable
159+
op.alter_column(
160+
'ds_app_link_data_source',
161+
'url_id',
162+
nullable=True
163+
)
164+
165+
166+
167+
def _alter_ds_app_link_meta_url_table():
168+
# Drop joint primary key for url_id and agency_id
169+
op.drop_constraint(
170+
'url_ds_meta_url_pkey',
171+
'ds_app_link_meta_url',
172+
type_='primary'
173+
)
174+
# Drop unique constraint for ds_meta_url_id
175+
op.drop_constraint(
176+
'url_ds_meta_url_ds_meta_url_id_key',
177+
'ds_app_link_meta_url',
178+
type_='unique'
179+
)
180+
# Drop agency_id column
181+
op.drop_column(
182+
'ds_app_link_meta_url',
183+
'agency_id'
184+
)
185+
# Make ds_meta_url_id primary key
186+
op.create_primary_key(
187+
'ds_app_link_meta_url_pkey',
188+
'ds_app_link_meta_url',
189+
['ds_meta_url_id']
190+
)
191+
# Add unique constraint for url_id
192+
op.create_unique_constraint(
193+
'uq_ds_app_link_meta_url_url_id',
194+
'ds_app_link_meta_url',
195+
['url_id']
196+
)
197+
# URL ID
198+
## Drop foreign key
199+
op.drop_constraint(
200+
'url_ds_meta_url_url_id_fkey',
201+
'ds_app_link_meta_url',
202+
type_='foreignkey'
203+
)
204+
## Recreate foreign key with ON DELETE SET NULL
205+
op.create_foreign_key(
206+
'ds_app_link_meta_url_url_id_fkey',
207+
'ds_app_link_meta_url',
208+
'urls',
209+
['url_id'],
210+
['id'],
211+
ondelete='SET NULL'
212+
)
213+
## Alter url_id column to be nullable
214+
op.alter_column(
215+
'ds_app_link_meta_url',
216+
'url_id',
217+
nullable=True
218+
)
219+
220+
221+
def _add_flag_deletion_tables():
222+
op.create_table(
223+
'flag_ds_delete_agency',
224+
sa.Column(
225+
'ds_agency_id',
226+
sa.Integer(),
227+
sa.ForeignKey(
228+
'ds_app_link_agency.ds_agency_id',
229+
ondelete='CASCADE'
230+
),
231+
primary_key=True,
232+
nullable=False
233+
),
234+
created_at_column()
235+
)
236+
237+
op.create_table(
238+
'flag_ds_delete_data_source',
239+
sa.Column(
240+
'ds_data_source_id',
241+
sa.Integer(),
242+
sa.ForeignKey(
243+
'ds_app_link_data_source.ds_data_source_id',
244+
ondelete='CASCADE'
245+
),
246+
primary_key=True,
247+
nullable=False
248+
),
249+
created_at_column(),
250+
)
251+
252+
op.create_table(
253+
'flag_ds_delete_meta_url',
254+
sa.Column(
255+
'ds_meta_url_id',
256+
sa.Integer(),
257+
sa.ForeignKey(
258+
'ds_app_link_meta_url.ds_meta_url_id',
259+
ondelete='CASCADE'
260+
),
261+
primary_key=True,
262+
nullable=False
263+
),
264+
created_at_column(),
265+
)
266+
267+
268+
def _rename_existing_tables_to_ds_app_format():
269+
op.rename_table(
270+
'url_data_source',
271+
'ds_app_link_data_source'
272+
)
273+
op.rename_table(
274+
'url_ds_meta_url',
275+
'ds_app_link_meta_url'
276+
)
277+
278+
def _migrate_agency_ids_to_ds_agency_link():
279+
"""
280+
While this migration uses the existing DS agency IDs for both sm and ds agency ids
281+
From this point onward the sm ID is internal to the SM application,
282+
and the same is true for DS ID.
283+
"""
284+
285+
op.execute("""
286+
INSERT INTO ds_app_link_agency(agency_id, ds_agency_id)
287+
SELECT agency_id, agency_id
288+
FROM agencies
289+
""")
290+
291+
292+
def remove_id_column_from_agencies():
293+
op.drop_column(
294+
'agencies',
295+
'id'
296+
)
297+
298+
def rename_agency_id_to_id():
299+
op.alter_column(
300+
'agencies',
301+
'agency_id',
302+
new_column_name='id'
303+
)
304+
305+
def _create_ds_agency_link():
306+
op.create_table(
307+
'ds_app_link_agency',
308+
sa.Column(
309+
'agency_id',
310+
sa.Integer(),
311+
sa.ForeignKey(
312+
'agencies.agency_id',
313+
ondelete='SET NULL'
314+
),
315+
nullable=True
316+
),
317+
sa.Column(
318+
'ds_agency_id',
319+
sa.Integer(),
320+
nullable=False,
321+
primary_key=True
322+
),
323+
created_at_column(),
324+
last_synced_at_column(),
325+
sa.UniqueConstraint(
326+
"agency_id", name="uq_ds_app_link_agency_agency_id"
327+
)
328+
)
329+
330+
331+
def _create_sync_log():
332+
op.create_table(
333+
'sync_log',
334+
sa.Column(
335+
'resource_type',
336+
sa.Enum(
337+
'agency',
338+
'data_source',
339+
'meta_url',
340+
name='resource_type_enum'
341+
),
342+
nullable=False,
343+
),
344+
sa.Column(
345+
'sync_type',
346+
sa.Enum(
347+
'add',
348+
'update',
349+
'delete',
350+
name='sync_type_enum'
351+
),
352+
nullable=False,
353+
),
354+
sa.Column(
355+
'count',
356+
sa.Integer(),
357+
nullable=False,
358+
),
359+
created_at_column(),
360+
sa.PrimaryKeyConstraint(
361+
'resource_type',
362+
'sync_type',
363+
'created_at'
364+
)
365+
)
366+
367+
368+
def downgrade() -> None:
369+
pass

0 commit comments

Comments
 (0)