Skip to content

Commit

Permalink
feat: migrate to new Events and new event-driven architecture
Browse files Browse the repository at this point in the history
  • Loading branch information
@aleortega
aleortega committed Oct 11, 2024
1 parent 0e20c50 commit 4973ac8
Show file tree
Hide file tree
Showing 5 changed files with 161 additions and 72 deletions.
5 changes: 3 additions & 2 deletions .env.default
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
# 2. `.env` file contents
# 3. `.env.default` file contents.

AWS_REGION=us-east-1
SNS_ARN=
SNS_ARN=''
SNS_ENDPOINT=''
CONTENT_SERVER_URLS=''
HTTP_SERVER_PORT=3000
HTTP_SERVER_HOST=0.0.0.0
7 changes: 4 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@
"dependencies": {
"@aws-sdk/client-sns": "^3.616.0",
"@dcl/catalyst-storage": "^2.0.3",
"@dcl/schemas": "^6.11.1",
"@dcl/schemas": "https://sdk-team-cdn.decentraland.org/@dcl/schemas/branch/feat/catalystdeployment-event-enhance/dcl-schemas-14.1.1-11298178941.commit-64eb184.tgz",
"@dcl/snapshots-fetcher": "^5.0.4",
"@well-known-components/env-config-provider": "^1.1.1",
"@well-known-components/http-server": "^1.1.1",
"@well-known-components/interfaces": "^1.2.0",
"@well-known-components/logger": "^3.1.2",
"@well-known-components/metrics": "^2.0.1"
"@well-known-components/metrics": "^2.0.1",
"dcl-catalyst-client": "^21.7.0"
}
}
}
108 changes: 46 additions & 62 deletions src/adapters/deployer/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { downloadEntityAndContentFiles } from '@dcl/snapshots-fetcher'
import { IDeployerComponent } from '@dcl/snapshots-fetcher/dist/types'
import { PublishCommand, SNSClient } from '@aws-sdk/client-sns'
import { AppComponents } from '../../types'
import { DeploymentToSqs } from '@dcl/schemas/dist/misc/deployments-to-sqs'
import { createContentClient } from 'dcl-catalyst-client'
import { CatalystDeploymentEvent, Entity, Events } from '@dcl/schemas'

export function createDeployerComponent(
components: Pick<AppComponents, 'logs' | 'storage' | 'downloadQueue' | 'fetch' | 'metrics' | 'sns'>
Expand All @@ -15,84 +15,68 @@ export function createDeployerComponent(

return {
async deployEntity(entity, servers) {
const markAsDeployed = entity.markAsDeployed ? entity.markAsDeployed : async () => {}
try {
const exists = await components.storage.exist(entity.entityId)

const isSnsEntityToSend =
(entity.entityType === 'scene' || entity.entityType === 'wearable' || entity.entityType === 'emote') &&
!!components.sns.arn
const markEntityAsDeployed = entity.markAsDeployed ? entity.markAsDeployed : async () => {}

const isSnsEventToSend = !!components.sns.eventArn
try {
const contentClient = createContentClient({ fetcher: components.fetch, url: servers[0] })
const fetchedEntity: Entity | undefined = await contentClient.fetchEntityById(entity.entityId)

if (exists) {
return await markAsDeployed()
if (!fetchedEntity) {
logger.info('Entity not found on Catalyst, marking it as deployed', { entityId: entity.entityId })
await markEntityAsDeployed()
}

const deploymentToSqs: DeploymentToSqs = {
entity,
logger.info('Entity downloaded correctly from Catalyst', {
entityId: entity.entityId,
entityType: entity.entityType,
pointers: fetchedEntity.pointers.join(' - ')
})

const eventToPublish: CatalystDeploymentEvent = {
type: Events.Type.CATALYST_DEPLOYMENT,
subType: entity.entityType as Events.SubType.CatalystDeployment,
key: entity.entityId,
timestamp: fetchedEntity.timestamp,
entity: fetchedEntity,
contentServerUrls: servers
}

if (isSnsEntityToSend) {
await components.downloadQueue.onSizeLessThan(1000)

void components.downloadQueue.scheduleJob(async () => {
logger.info('Downloading entity', {
entityId: entity.entityId,
entityType: entity.entityType,
servers: servers.join(',')
})

await downloadEntityAndContentFiles(
{ ...components, fetcher: components.fetch },
entity.entityId,
servers,
new Map(),
'content',
10,
1000
)

logger.info('Entity stored', { entityId: entity.entityId, entityType: entity.entityType })
// send sns
const receipt = await client.send(
new PublishCommand({
TopicArn: components.sns.arn,
Message: JSON.stringify(deploymentToSqs)
})
)
logger.info('Notification sent', {
messageId: receipt.MessageId as any,
sequenceNumber: receipt.SequenceNumber as any
})
const receipt = await client.send(
new PublishCommand({
TopicArn: components.sns.arn,
Message: JSON.stringify(eventToPublish)
})
}
)

if (isSnsEventToSend) {
const receipt = await client.send(
new PublishCommand({
TopicArn: components.sns.eventArn,
Message: JSON.stringify(deploymentToSqs)
})
)
logger.info('Notification sent to events SNS', {
messageId: receipt.MessageId as any,
sequenceNumber: receipt.SequenceNumber as any,
entityId: deploymentToSqs.entity.entityId
})
}
logger.info('Notification sent', {
messageId: receipt.MessageId as any,
sequenceNumber: receipt.SequenceNumber as any,
entityId: entity.entityId,
entityPointers: fetchedEntity.pointers.join(' - ')
})

await markAsDeployed()
await markEntityAsDeployed()
} catch (error: any) {
logger.error('Failure while processing entity', {
entityId: entity.entityId,
entityType: entity.entityType,
error: error?.message
})

logger.debug('Error details', {
entityId: entity.entityId,
stack: error.stack,
message: error.message
})

const isNotRetryable = /status: 4\d{2}/.test(error.message)
if (isNotRetryable) {
logger.error('Failed to download entity', {
entityId: entity.entityId,
entityType: entity.entityType,
error: error?.message
})
await markAsDeployed()
await markEntityAsDeployed()
}
}
},
Expand Down
2 changes: 0 additions & 2 deletions src/components.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ export async function initComponents(): Promise<AppComponents> {

const bucket = await config.getString('BUCKET')
const snsArn = await config.getString('SNS_ARN')
const eventSnsArn = await config.getString('EVENTS_SNS_ARN')
const optionalSnsEndpoint = await config.getString('SNS_ENDPOINT')

const storage = bucket
Expand All @@ -49,7 +48,6 @@ export async function initComponents(): Promise<AppComponents> {

const sns: SnsComponent = {
arn: snsArn,
eventArn: eventSnsArn,
optionalSnsEndpoint
}

Expand Down
111 changes: 108 additions & 3 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,11 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

"@dcl/catalyst-contracts@^4.4.0":
version "4.4.2"
resolved "https://registry.yarnpkg.com/@dcl/catalyst-contracts/-/catalyst-contracts-4.4.2.tgz#e7a329d7aee25c90d7ee07dbe82200a048f17ff9"
integrity sha512-gJZo3IB8U+jhBJWR0DgoLS+zaUDIb/u79e7Rp+MEM78uH3bvC19S3Dd8lxWEbPXNKlCB0saUptfK/Buw0c1y2Q==

"@dcl/catalyst-storage@^2.0.0", "@dcl/catalyst-storage@^2.0.3":
version "2.0.4"
resolved "https://registry.yarnpkg.com/@dcl/catalyst-storage/-/catalyst-storage-2.0.4.tgz#eca79f335653e073cbec31d13dba5a3c1e06cf97"
Expand All @@ -751,6 +756,15 @@
aws-sdk "^2.1407.0"
destroy "^1.2.0"

"@dcl/crypto@^3.4.0":
version "3.4.5"
resolved "https://registry.yarnpkg.com/@dcl/crypto/-/crypto-3.4.5.tgz#95ca2beab46fa3494b4d38f009bf0d49c87ba235"
integrity sha512-uneyjOAOx7pi5kabZsLmPm9kSLkCk4Cok8FUsvT+6k8RquqkjKqocvkGVOMaoWsfU6S3mkLOyaeqEKmOy4ErxA==
dependencies:
"@dcl/schemas" "^9.2.0"
eth-connect "^6.0.3"
ethereum-cryptography "^1.0.3"

"@dcl/eslint-config@^1.1.12":
version "1.1.12"
resolved "https://registry.yarnpkg.com/@dcl/eslint-config/-/eslint-config-1.1.12.tgz#13ceda9fafc0aad48796f006f81c5da91c1cb6ff"
Expand All @@ -777,7 +791,32 @@
ipfs-unixfs-importer "^7.0.3"
multiformats "^9.6.3"

"@dcl/schemas@^6.11.1", "@dcl/schemas@^6.2.0":
"@dcl/hashing@^3.0.0":
version "3.0.4"
resolved "https://registry.yarnpkg.com/@dcl/hashing/-/hashing-3.0.4.tgz#4df2a4cb3a8114765aed34cb57b91c93bf33bfb3"
integrity sha512-Cg+MoIOn+BYmQV2q8zSFnNYY+GldlnUazwBnfgrq3i66ZxOaZ65h01btd8OUtSAlfWG4VTNIOHDjtKqmuwJNBg==

"@dcl/schemas@^11.5.0":
version "11.12.0"
resolved "https://registry.yarnpkg.com/@dcl/schemas/-/schemas-11.12.0.tgz#f52493b615ebdea8e02819bc2ed4a456239f3747"
integrity sha512-L04KTucvxSnrHDAl3/rnkzhjfZ785dSSPeKarBVfzyuw41uyQ0Mh4HVFWjX9hC+f/nMpM5Adg5udlT5efmepcA==
dependencies:
ajv "^8.11.0"
ajv-errors "^3.0.0"
ajv-keywords "^5.1.0"
mitt "^3.0.1"

"@dcl/schemas@^14.1.0":
version "14.1.0"
resolved "https://registry.yarnpkg.com/@dcl/schemas/-/schemas-14.1.0.tgz#f138401748cedfda6c3d618265368405386be2cc"
integrity sha512-KE7679WNJqlrXH088pvXh2a6YLtDL+PvZYXY8ZXShhEJrnxjUhXGYH1LJckPLy4Tb7GEJBwaCcfYlVVVMrDLeA==
dependencies:
ajv "^8.11.0"
ajv-errors "^3.0.0"
ajv-keywords "^5.1.0"
mitt "^3.0.1"

"@dcl/schemas@^6.2.0":
version "6.19.0"
resolved "https://registry.yarnpkg.com/@dcl/schemas/-/schemas-6.19.0.tgz#42cfe23226782a736e3956ebcbd39ef21e63aeb1"
integrity sha512-S8lrq8L1vriVXkBzeycxppjbfgJ5XofA9pbCKdteJR+79r6Dn5oLo3t5g7RcMQDihdjWdDLbbwxlEAkPgmvc8w==
Expand All @@ -786,6 +825,24 @@
ajv-errors "^3.0.0"
ajv-keywords "^5.1.0"

"@dcl/schemas@^9.2.0":
version "9.15.0"
resolved "https://registry.yarnpkg.com/@dcl/schemas/-/schemas-9.15.0.tgz#81337faa396d21a2d1e704e5ec3cfd7b7b14343e"
integrity sha512-nip5rsOcJplNfBWeImwezuHLprM0gLW03kEeqGIvT9J6HnEBTtvIwkk9+NSt7hzFKEvWGI+C23vyNWbG3nU+SQ==
dependencies:
ajv "^8.11.0"
ajv-errors "^3.0.0"
ajv-keywords "^5.1.0"

"@dcl/schemas@https://sdk-team-cdn.decentraland.org/@dcl/schemas/branch/feat/catalystdeployment-event-enhance/dcl-schemas-14.1.1-11298178941.commit-64eb184.tgz":
version "14.1.1-11298178941.commit-64eb184"
resolved "https://sdk-team-cdn.decentraland.org/@dcl/schemas/branch/feat/catalystdeployment-event-enhance/dcl-schemas-14.1.1-11298178941.commit-64eb184.tgz#60ef6f3512e99b8a313372879fc0d1dc6a0cea39"
dependencies:
ajv "^8.11.0"
ajv-errors "^3.0.0"
ajv-keywords "^5.1.0"
mitt "^3.0.1"

"@dcl/snapshots-fetcher@^5.0.4":
version "5.0.4"
resolved "https://registry.yarnpkg.com/@dcl/snapshots-fetcher/-/snapshots-fetcher-5.0.4.tgz#300f937125c6af7988ab9007e70580eab656c6c8"
Expand Down Expand Up @@ -1960,6 +2017,14 @@
dependencies:
dotenv "^16.0.1"

"@well-known-components/fetch-component@^2.0.0":
version "2.0.2"
resolved "https://registry.yarnpkg.com/@well-known-components/fetch-component/-/fetch-component-2.0.2.tgz#02260611f9d05551efe825c4358cbc628a71c4c0"
integrity sha512-LdY+6n9kuyACg3fcU4qMrNhLZuG7eqPxLSqzDgQyoHKeNjlzggoUqTVJKtIyi6vjPs8pSQ/Fx1xdLuBhOKCgww==
dependencies:
"@well-known-components/interfaces" "^1.4.1"
cross-fetch "^3.1.5"

"@well-known-components/http-server@^1.1.1":
version "1.1.6"
resolved "https://registry.yarnpkg.com/@well-known-components/http-server/-/http-server-1.1.6.tgz#3abc208820512bfc98080e85a4f4f6217ca9d810"
Expand Down Expand Up @@ -1987,6 +2052,15 @@
"@types/node-fetch" "^2.5.12"
typed-url-params "^1.0.1"

"@well-known-components/interfaces@^1.4.1":
version "1.4.3"
resolved "https://registry.yarnpkg.com/@well-known-components/interfaces/-/interfaces-1.4.3.tgz#026047966db480bd561a315fdc8116b88bf8d27a"
integrity sha512-roVtoOHG6uaH+nL4C0ISnAwkkopc2FLsS7fqX+roI22EdX9PAknPoImhPU8/3u6jgRAVpglX5Zj4nWZkSaXPkQ==
dependencies:
"@types/node" "^20.3.1"
"@types/node-fetch" "^2.5.12"
typed-url-params "^1.0.1"

"@well-known-components/logger@^3.1.2":
version "3.1.3"
resolved "https://registry.yarnpkg.com/@well-known-components/logger/-/logger-3.1.3.tgz#b1f4a76ac5e0d89c276cf339fe44a61f46023d2f"
Expand Down Expand Up @@ -2498,7 +2572,7 @@ [email protected]:
resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
integrity sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==

[email protected]:
[email protected], cookie@^0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b"
integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==
Expand All @@ -2524,6 +2598,13 @@ create-jest@^29.7.0:
jest-util "^29.7.0"
prompts "^2.0.1"

cross-fetch@^3.1.5:
version "3.1.8"
resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.8.tgz#0327eba65fd68a7d119f8fb2bf9334a1a7956f82"
integrity sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==
dependencies:
node-fetch "^2.6.12"

cross-spawn@^7.0.2, cross-spawn@^7.0.3:
version "7.0.3"
resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
Expand All @@ -2533,6 +2614,20 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3:
shebang-command "^2.0.0"
which "^2.0.1"

dcl-catalyst-client@^21.7.0:
version "21.7.0"
resolved "https://registry.yarnpkg.com/dcl-catalyst-client/-/dcl-catalyst-client-21.7.0.tgz#dbf33dff0bcf382c8383f359dbfaf7f85011af25"
integrity sha512-10NyeYrKSh7yM5y7suLfoDeVAM9xknlvlxDBof1lJiuPYPzj5Aee8kaEDfXzVWTpfI7ssvSXhBlGVRvyd0RcJA==
dependencies:
"@dcl/catalyst-contracts" "^4.4.0"
"@dcl/crypto" "^3.4.0"
"@dcl/hashing" "^3.0.0"
"@dcl/schemas" "^11.5.0"
"@well-known-components/fetch-component" "^2.0.0"
cookie "^0.5.0"
cross-fetch "^3.1.5"
form-data "^4.0.0"

[email protected]:
version "2.6.9"
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
Expand Down Expand Up @@ -2907,6 +3002,11 @@ etag@~1.8.1:
resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
integrity sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==

eth-connect@^6.0.3:
version "6.2.4"
resolved "https://registry.yarnpkg.com/eth-connect/-/eth-connect-6.2.4.tgz#81a94bce29f6743b46f336a0ec9d42ed73fbe6fd"
integrity sha512-K0+g+pZoWkcJKKc4hwlYvaruoMBFcARoULIdf50bz/Zk9YdgFoKPjlRQWAtBgSDfxJS7AAHqg40k2Z/uQI0aNw==

ethereum-cryptography@^1.0.3:
version "1.2.0"
resolved "https://registry.yarnpkg.com/ethereum-cryptography/-/ethereum-cryptography-1.2.0.tgz#5ccfa183e85fdaf9f9b299a79430c044268c9b3a"
Expand Down Expand Up @@ -4236,6 +4336,11 @@ minimist@^1.2.5:
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c"
integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==

mitt@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/mitt/-/mitt-3.0.1.tgz#ea36cf0cc30403601ae074c8f77b7092cdab36d1"
integrity sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==

[email protected]:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
Expand Down Expand Up @@ -4323,7 +4428,7 @@ nise@^5.1.5:
just-extend "^4.0.2"
path-to-regexp "^1.7.0"

node-fetch@^2.6.1, node-fetch@^2.6.7:
node-fetch@^2.6.1, node-fetch@^2.6.12, node-fetch@^2.6.7:
version "2.7.0"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
Expand Down

0 comments on commit 4973ac8

Please sign in to comment.