-
Notifications
You must be signed in to change notification settings - Fork 0
/
ingest.js
166 lines (133 loc) · 4.6 KB
/
ingest.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
const config = require("./config");
const utils = require("./utils");
const mongo = require("./mongo");
mongo.connect(config.dbUrl, config.dbName, async () => {
const startTime = new Date();
let creatorArray = [];
let levelArray = [];
let levels = [];
utils.writeToLogFile("Starting ingest");
await mongo.models.thresholds.updateThreshold();
// Get levels from MMM's popular API
const popularLevels = await utils.api
.getPopularLevels()
.catch(utils.handleError);
// Get the user IDs for the popular levels
const popularLevelCreatorIDs = utils.getCreatorsFromArray(popularLevels);
// Get the ids of creators (from the popular API) that aren't in the db
const nonExistentCreatorIds = await mongo.models.creators
.getNonExistentCreatorIDs(popularLevelCreatorIDs)
.catch(utils.handleError);
utils.writeToLogFile(
`# of creators from the popular API to add: ${nonExistentCreatorIds.length}`
);
// Add the ids from above to the queue
await Promise.all(
nonExistentCreatorIds.map(e => {
return mongo.models.queue.addToQueue("creator", e);
})
).catch(utils.handleError);
// Get the outdated creators in the DB
const outdatedCreators = await mongo.models.creators
.getOutdatedCreators()
.catch(utils.handleError);
// Add the outdated creators to the queue
await Promise.all(
outdatedCreators.map(e => {
return mongo.models.queue.addToQueue("creator", e.id);
})
).catch(utils.handleError);
// Get the creators from the queue
const queuedCreators = await mongo.models.queue
.getQueuedCreators()
.catch(utils.handleError);
// Remove any possible duplicates
const queuedCreatorIDs = queuedCreators.map(e => e.id);
const uniqueQueuedCreatorIDs = queuedCreatorIDs.filter(
(e, i) => queuedCreatorIDs.indexOf(e) === i
);
utils.writeToLogFile(
`# of creators queued: ${uniqueQueuedCreatorIDs.length}`
);
// Get the creators from the MMM API
for (let id of uniqueQueuedCreatorIDs) {
creatorArray.push(
await utils.api.getCreatorData(id).catch(utils.handleError)
);
}
// Get an array of user icons
const icons = creatorArray.map(e => {
return e.icon;
});
// Remove any duplicates from the icon array
const iconsNoDups = icons.filter((e, i) => icons.indexOf(e) === i);
// Add any icons to the db if they don't exist already
for (let id of iconsNoDups) {
if (id) {
await mongo.models.icons.addUserAvatar(id).catch(utils.handleError);
}
}
// Update the creators collection
await Promise.all(
creatorArray.map(e => {
return mongo.models.creators.updateCreator(e);
})
).catch(utils.handleError);
// Get the levels from the creator array so we don't have to double the load on the API
creatorArray.forEach(e => {
levels = levels.concat(e.levels);
});
// Update levels in the db
await Promise.all(
levels.map(e => {
if (e) {
return mongo.models.levels.updateLevel(e);
}
})
).catch(utils.handleError);
// Get outdated levels
const outdatedLevels = await mongo.models.levels
.getOutdatedLevels()
.catch(utils.handleError);
// Add outdated Levels to the queue
await Promise.all(
outdatedLevels.map(e => {
return mongo.models.queue.addToQueue("level", e.id);
})
).catch(utils.handleError);
// Get queued Levels from the queue collection
const queuedLevels = await mongo.models.queue
.getQueuedLevels()
.catch(utils.handleError);
// Remove any possible duplicates
const queuedLevelIDs = queuedLevels.map(e => e.id);
const uniqueQueuedLevelIDs = queuedLevelIDs.filter(
(e, i) => queuedLevelIDs.indexOf(e) === i
);
utils.writeToLogFile(`# of levels queued: ${uniqueQueuedLevelIDs.length}`);
// Get the creators from the MMM API
for (let id of uniqueQueuedLevelIDs) {
levelArray.push(await utils.api.getLevelData(id).catch(utils.handleError));
}
// Get an array of all the boss icons from the oudated levels array
const bossIcons = levelArray.map(e => {
// level might be deleted, check for boss value
if (e.boss) return e.boss;
});
// Remove any duplicates from the boss icons array
const bossIconsNoDups = bossIcons.filter(
(e, i) => bossIcons.indexOf(e) === i
);
// Add any boss icons that don't already exist in the db
for (let id of bossIconsNoDups) {
await mongo.models.icons.addBossAvatar(id).catch(utils.handleError);
}
// Update levels in the db
await Promise.all(
levelArray.map(e => {
return mongo.models.levels.updateLevel(e);
})
).catch(utils.handleError);
utils.writeToLogFile(`Finished ingest`);
process.exit();
});