Skip to content
This repository was archived by the owner on Jul 1, 2024. It is now read-only.

Commit a7453a0

Browse files
kullakulla
committed
feat: Simplify event tables
Fixes #344
1 parent 224e9ff commit a7453a0

File tree

1 file changed

+363
-0
lines changed

1 file changed

+363
-0
lines changed

‎src/20240607193000-simplification-event-of-tables.ts

+363
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
import * as t from 'io-ts'
2+
3+
import { ApiCache, Database, SlackLogger } from './utils'
4+
5+
export async function up(db: Database) {
6+
const apiCache = new ApiCache()
7+
const logger = new SlackLogger(
8+
'20240607193000-simplification-event-of-tables',
9+
)
10+
11+
console.log(
12+
'Starting migration 20240607193000-simplification-event-of-tables',
13+
)
14+
15+
for (const columnName of ['uuid_parameter', 'uuid_parameter2']) {
16+
await db.runSql(`ALTER TABLE event_log ADD ${columnName} BIGINT NULL;`)
17+
await db.runSql(
18+
`CREATE INDEX idx_event_${columnName} ON event_log (${columnName});`,
19+
)
20+
await db.runSql(`
21+
ALTER TABLE event_log
22+
ADD CONSTRAINT fk_event_${columnName}
23+
FOREIGN KEY (${columnName})
24+
REFERENCES uuid(id)
25+
ON DELETE CASCADE;
26+
`)
27+
console.log(`Column ${columnName} added to event_log`)
28+
}
29+
await db.runSql('ALTER TABLE event_log ADD string_parameter MEDIUMTEXT NULL;')
30+
console.log('Column string_parameter added to event_log')
31+
32+
await updateEventTable(db, logger)
33+
34+
await db.runSql('ALTER TABLE event RENAME TO event_type;')
35+
await db.runSql(
36+
'ALTER TABLE event_log RENAME COLUMN event_id TO event_type_id;',
37+
)
38+
console.log('Table event renamed to event_type')
39+
40+
await db.runSql('alter table event_log rename to event')
41+
await db.runSql(
42+
'alter table notification_event rename column event_log_id to event_id',
43+
)
44+
console.log('Table event_log renamed to event')
45+
46+
// TODO: Log those tables before dropping them
47+
await db.runSql('drop table event_parameter_string')
48+
await db.runSql('drop table event_parameter_uuid')
49+
await db.runSql('drop table event_parameter')
50+
await db.runSql('drop table event_parameter_name')
51+
52+
await logger.closeAndSend()
53+
// To reduce the time between deleting the keys and finishing the DB
54+
// transaction, this should be the last command
55+
await apiCache.deleteKeysAndQuit()
56+
}
57+
58+
enum EventType {
59+
ArchiveThread = 'discussion/comment/archive',
60+
RestoreThread = 'discussion/restore',
61+
CreateComment = 'discussion/comment/create',
62+
CreateThread = 'discussion/create',
63+
CreateEntity = 'entity/create',
64+
SetLicense = 'license/object/set',
65+
CreateEntityLink = 'entity/link/create',
66+
RemoveEntityLink = 'entity/link/remove',
67+
CreateEntityRevision = 'entity/revision/add',
68+
CheckoutRevision = 'entity/revision/checkout',
69+
RejectRevision = 'entity/revision/reject',
70+
CreateTaxonomyLink = 'taxonomy/term/associate',
71+
RemoveTaxonomyLink = 'taxonomy/term/dissociate',
72+
CreateTaxonomyTerm = 'taxonomy/term/create',
73+
SetTaxonomyTerm = 'taxonomy/term/update',
74+
SetTaxonomyParent = 'taxonomy/term/parent/change',
75+
RestoreUuid = 'uuid/restore',
76+
TrashUuid = 'uuid/trash',
77+
}
78+
79+
const DatabaseEventRepresentations = {
80+
ArchiveThread: getDatabaseRepresentationDecoder({
81+
type: EventType.ArchiveThread,
82+
uuidParameters: t.type({}),
83+
stringParameters: t.type({}),
84+
}),
85+
RestoreThread: getDatabaseRepresentationDecoder({
86+
type: EventType.RestoreThread,
87+
uuidParameters: t.type({}),
88+
stringParameters: t.type({}),
89+
}),
90+
CreateComment: getDatabaseRepresentationDecoder({
91+
type: EventType.CreateComment,
92+
uuidParameters: t.type({ discussion: t.number }),
93+
stringParameters: t.type({}),
94+
}),
95+
CreateThread: getDatabaseRepresentationDecoder({
96+
type: EventType.CreateThread,
97+
uuidParameters: t.type({ on: t.number }),
98+
stringParameters: t.type({}),
99+
}),
100+
CreateEntity: getDatabaseRepresentationDecoder({
101+
type: EventType.CreateEntity,
102+
uuidParameters: t.type({}),
103+
stringParameters: t.type({}),
104+
}),
105+
SetLicense: getDatabaseRepresentationDecoder({
106+
type: EventType.SetLicense,
107+
uuidParameters: t.type({}),
108+
stringParameters: t.type({}),
109+
}),
110+
CreateEntityLink: getDatabaseRepresentationDecoder({
111+
type: EventType.CreateEntityLink,
112+
uuidParameters: t.type({ parent: t.number }),
113+
stringParameters: t.type({}),
114+
}),
115+
RemoveEntityLink: getDatabaseRepresentationDecoder({
116+
type: EventType.RemoveEntityLink,
117+
uuidParameters: t.type({ parent: t.number }),
118+
stringParameters: t.type({}),
119+
}),
120+
CreateEntityRevision: getDatabaseRepresentationDecoder({
121+
type: EventType.CreateEntityRevision,
122+
uuidParameters: t.type({ repository: t.number }),
123+
stringParameters: t.type({}),
124+
}),
125+
CheckoutRevision: getDatabaseRepresentationDecoder({
126+
type: EventType.CheckoutRevision,
127+
uuidParameters: t.type({ repository: t.number }),
128+
stringParameters: t.type({ reason: t.string }),
129+
}),
130+
RejectRevision: getDatabaseRepresentationDecoder({
131+
type: EventType.RejectRevision,
132+
uuidParameters: t.type({ repository: t.number }),
133+
stringParameters: t.type({ reason: t.string }),
134+
}),
135+
CreateTaxonomyLink: getDatabaseRepresentationDecoder({
136+
type: EventType.CreateTaxonomyLink,
137+
uuidParameters: t.type({ object: t.number }),
138+
stringParameters: t.type({}),
139+
}),
140+
RemoveTaxonomyLink: getDatabaseRepresentationDecoder({
141+
type: EventType.RemoveTaxonomyLink,
142+
uuidParameters: t.type({ object: t.number }),
143+
stringParameters: t.type({}),
144+
}),
145+
CreateTaxonomyTerm: getDatabaseRepresentationDecoder({
146+
type: EventType.CreateTaxonomyTerm,
147+
uuidParameters: t.type({}),
148+
stringParameters: t.type({}),
149+
}),
150+
SetTaxonomyTerm: getDatabaseRepresentationDecoder({
151+
type: EventType.SetTaxonomyTerm,
152+
uuidParameters: t.type({}),
153+
stringParameters: t.type({}),
154+
}),
155+
SetTaxonomyParent: getDatabaseRepresentationDecoder({
156+
type: EventType.SetTaxonomyParent,
157+
uuidParameters: t.type({
158+
from: t.union([t.number, t.null]),
159+
to: t.union([t.number, t.null]),
160+
}),
161+
stringParameters: t.type({}),
162+
}),
163+
TrashUuid: getDatabaseRepresentationDecoder({
164+
type: EventType.TrashUuid,
165+
uuidParameters: t.type({}),
166+
stringParameters: t.type({}),
167+
}),
168+
RestoreUuid: getDatabaseRepresentationDecoder({
169+
type: EventType.RestoreUuid,
170+
uuidParameters: t.type({}),
171+
stringParameters: t.type({}),
172+
}),
173+
} as const
174+
175+
type DatabaseEventRepresentation = {
176+
[P in keyof typeof DatabaseEventRepresentations]: t.TypeOf<
177+
(typeof DatabaseEventRepresentations)[P]
178+
>
179+
}[keyof typeof DatabaseEventRepresentations]
180+
181+
const DatabaseEventRepresentation: t.Type<DatabaseEventRepresentation> =
182+
t.union([
183+
DatabaseEventRepresentations.ArchiveThread,
184+
DatabaseEventRepresentations.CheckoutRevision,
185+
DatabaseEventRepresentations.CreateComment,
186+
DatabaseEventRepresentations.CreateEntity,
187+
DatabaseEventRepresentations.CreateEntityLink,
188+
DatabaseEventRepresentations.CreateEntityRevision,
189+
DatabaseEventRepresentations.CreateTaxonomyTerm,
190+
DatabaseEventRepresentations.CreateTaxonomyLink,
191+
DatabaseEventRepresentations.CreateThread,
192+
DatabaseEventRepresentations.RejectRevision,
193+
DatabaseEventRepresentations.RemoveEntityLink,
194+
DatabaseEventRepresentations.RemoveTaxonomyLink,
195+
DatabaseEventRepresentations.RestoreThread,
196+
DatabaseEventRepresentations.RestoreUuid,
197+
DatabaseEventRepresentations.SetLicense,
198+
DatabaseEventRepresentations.SetTaxonomyParent,
199+
DatabaseEventRepresentations.SetTaxonomyTerm,
200+
DatabaseEventRepresentations.TrashUuid,
201+
])
202+
203+
async function updateEventTable(db: Database, logger: SlackLogger) {
204+
interface Row {
205+
id: number
206+
}
207+
208+
const batchSize = 1000
209+
let offset = 0
210+
let rows: Row[] = []
211+
212+
console.log('Updating event table')
213+
214+
while (true) {
215+
console.log(`Processing rows from ${offset} to ${offset + batchSize}`)
216+
rows = await db.runSql<Row[]>(
217+
`
218+
select
219+
event_log.id as id,
220+
event.name as type,
221+
JSON_OBJECTAGG(
222+
COALESCE(event_parameter_name.name, "__unused"),
223+
event_parameter_uuid.uuid_id
224+
) as uuidParameters,
225+
JSON_OBJECTAGG(
226+
COALESCE(event_parameter_name.name, "__unused"),
227+
event_parameter_string.value
228+
) as stringParameters
229+
from event_log
230+
join event on event_log.event_id = event.id
231+
left join event_parameter on event_parameter.log_id = event_log.id
232+
left join event_parameter_name on event_parameter.name_id = event_parameter_name.id
233+
left join event_parameter_string on event_parameter_string.event_parameter_id = event_parameter.id
234+
left join event_parameter_uuid on event_parameter_uuid.event_parameter_id = event_parameter.id
235+
group by event_log.id
236+
order by id
237+
limit ? offset ?
238+
`,
239+
batchSize,
240+
offset,
241+
)
242+
243+
for (const row of rows) {
244+
if (DatabaseEventRepresentation.is(row)) {
245+
const newData = toNewData(row)
246+
247+
await db.runSql(
248+
`
249+
update event_log
250+
set uuid_parameter = ?,
251+
uuid_parameter2 = ?,
252+
string_parameter = ?
253+
where id = ?
254+
`,
255+
newData.uuidParameter,
256+
newData.uuidParameter2,
257+
newData.stringParameter,
258+
row.id,
259+
)
260+
} else {
261+
// This event is malformed => we should delete it
262+
await db.runSql('delete from event_log where id = ?', row.id)
263+
logger.logEvent('deleteEvent', { id: row.id })
264+
console.log(`event ${row.id} deleted because it is malformed`)
265+
}
266+
}
267+
268+
if (rows.length === 0) {
269+
break
270+
}
271+
272+
offset += rows.length
273+
}
274+
275+
console.log('Event table updated')
276+
}
277+
278+
// Update event table row based on event type
279+
function toNewData(row: DatabaseEventRepresentation) {
280+
const base: NewData = {
281+
uuidParameter: null,
282+
uuidParameter2: null,
283+
stringParameter: null,
284+
}
285+
286+
switch (row.type) {
287+
case EventType.ArchiveThread:
288+
return { ...base }
289+
case EventType.RestoreThread:
290+
return { ...base }
291+
case EventType.CreateComment:
292+
return { ...base, uuidParameter: row.uuidParameters.discussion }
293+
case EventType.CreateThread:
294+
return { ...base, uuidParameter: row.uuidParameters.on }
295+
case EventType.CreateEntity:
296+
return { ...base }
297+
case EventType.SetLicense:
298+
return { ...base }
299+
case EventType.CreateEntityLink:
300+
return { ...base, uuidParameter: row.uuidParameters.parent }
301+
case EventType.RemoveEntityLink:
302+
return { ...base, uuidParameter: row.uuidParameters.parent }
303+
case EventType.CreateEntityRevision:
304+
return { ...base, uuidParameter: row.uuidParameters.repository }
305+
case EventType.CheckoutRevision:
306+
return {
307+
...base,
308+
uuidParameter: row.uuidParameters.repository,
309+
stringParameter: row.stringParameters.reason,
310+
}
311+
case EventType.RejectRevision:
312+
return {
313+
...base,
314+
uuidParameter: row.uuidParameters.repository,
315+
stringParameter: row.stringParameters.reason,
316+
}
317+
case EventType.CreateTaxonomyLink:
318+
return { ...base, uuidParameter: row.uuidParameters.object }
319+
case EventType.RemoveTaxonomyLink:
320+
return { ...base, uuidParameter: row.uuidParameters.object }
321+
case EventType.CreateTaxonomyTerm:
322+
return { ...base }
323+
case EventType.SetTaxonomyTerm:
324+
return { ...base }
325+
case EventType.SetTaxonomyParent:
326+
return {
327+
...base,
328+
uuidParameter: row.uuidParameters.from,
329+
uuidParameter2: row.uuidParameters.to,
330+
}
331+
case EventType.TrashUuid:
332+
return { ...base }
333+
case EventType.RestoreUuid:
334+
return { ...base }
335+
}
336+
}
337+
338+
interface NewData {
339+
uuidParameter: number | null
340+
uuidParameter2: number | null
341+
stringParameter: string | null
342+
}
343+
344+
function getDatabaseRepresentationDecoder<
345+
Type extends EventType,
346+
UuidParameters extends Record<string, number | null>,
347+
StringParameters extends Record<string, string>,
348+
>({
349+
type,
350+
uuidParameters,
351+
stringParameters,
352+
}: {
353+
type: Type
354+
uuidParameters: t.Type<UuidParameters>
355+
stringParameters: t.Type<StringParameters>
356+
}) {
357+
return t.type({
358+
id: t.number,
359+
type: t.literal(type),
360+
uuidParameters: uuidParameters,
361+
stringParameters: stringParameters,
362+
})
363+
}

0 commit comments

Comments
 (0)