feat: DNS reconciler engine

app/lib/dns.server.ts

@@ -17,6 +17,7 @@ import type {
  ChangeResourceRecordSetsResponse,
  GetChangeResponse,
  ListResourceRecordSetsResponse,
+ Change,
 } from '@aws-sdk/client-route-53';
 import { DnsRecordType } from '@prisma/client';
 import { z } from 'zod';
@@ -58,7 +59,7 @@ const toFQDN = (domain: string) => domain.replace(/\.?$/, '.');
  * dev, we create it on startup if not set.
  * @returns string - the AWS Hosted Zone ID to use
  */
-async function hostedZoneId() {
+async function getHostedZoneId() {
  if (!process.env.AWS_ROUTE53_HOSTED_ZONE_ID) {
  if (process.env.NODE_ENV === 'production') {
  throw new Error('AWS_ROUTE53_HOSTED_ZONE_ID environment variable is missing');
@@ -74,7 +75,7 @@ async function hostedZoneId() {
  return process.env.AWS_ROUTE53_HOSTED_ZONE_ID;
 }
 
-const credentials = () => {
+const getCredentials = () => {
  if (process.env.NODE_ENV === 'production') {
  return {
  accessKeyId: AWS_ACCESS_KEY_ID,
@@ -100,7 +101,7 @@ const awsEndpoint = () => {
 export const route53Client = new Route53Client({
  endpoint: awsEndpoint(),
  region: process.env.AWS_REGION || 'us-east-1',
- credentials: credentials(),
+ credentials: getCredentials(),
 });
 
 export async function createHostedZone(domain: string) {
@@ -138,7 +139,7 @@ export const createDnsRecord = (
 async function checkDnsRecordExists(type: DnsRecordType, fqdn: string, value: string) {
  try {
  const command = new ListResourceRecordSetsCommand({
- HostedZoneId: await hostedZoneId(),
+ HostedZoneId: await getHostedZoneId(),
  StartRecordName: fqdn,
  StartRecordType: type,
  MaxItems: 1,
@@ -165,6 +166,55 @@ async function checkDnsRecordExists(type: DnsRecordType, fqdn: string, value: st
  }
 }
 
+/**
+ * Get a full page of records from AWS Route53. If fqdn and recordType is specified,
+ * load the next page starting with that record
+ *
+ * https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/clients/client-route-53/classes/listresourcerecordsetscommand.html
+ */
+export const getDnsRecordSetPage = async (
+ fqdn?: string,
+ // Using string as it cam be any record type not just the ones we use, also AWS sdk refers to it as string
+ type?: string
+): Promise<ListResourceRecordSetsResponse> => {
+ try {
+ const command = new ListResourceRecordSetsCommand({
+ HostedZoneId: await getHostedZoneId(),
+ StartRecordName: fqdn,
+ StartRecordType: type,
+ });
+
+ return route53Client.send(command);
+ } catch (error) {
+ logger.warn('DNS Error - Error loading record page from Route53', { type, fqdn, error });
+
+ // Rethrow to keep stack
+ throw error;
+ }
+};
+
+/**
+ * Execute a complete changeset in Route53 generated by the reconciler
+ */
+export const executeChangeSet = async (Changes: Change[]) => {
+ try {
+ const command = new ChangeResourceRecordSetsCommand({
+ ChangeBatch: { Changes },
+ HostedZoneId: await getHostedZoneId(),
+ });
+ const response: ChangeResourceRecordSetsResponse = await route53Client.send(command);
+
+ if (!response.ChangeInfo?.Id) {
+ throw new Error(`DNS Error - missing ID in AWS ChangeInfo response`);
+ }
+
+ return response.ChangeInfo.Id;
+ } catch (error) {
+ logger.error('DNS Error - Failed to execute changeSet', { error });
+ throw error;
+ }
+};
+
 export const upsertDnsRecord = async (
  username: string,
  type: DnsRecordType,
@@ -210,7 +260,7 @@ export const upsertDnsRecord = async (
  },
  ],
  },
- HostedZoneId: await hostedZoneId(),
+ HostedZoneId: await getHostedZoneId(),
  });
  const response: ChangeResourceRecordSetsResponse = await route53Client.send(command);
 
@@ -278,7 +328,7 @@ export const deleteDnsRecord = async (
  },
  ],
  },
- HostedZoneId: await hostedZoneId(),
+ HostedZoneId: await getHostedZoneId(),
  });
 
  const response: ChangeResourceRecordSetsResponse = await route53Client.send(command);

app/models/dns-record.server.ts

@@ -126,3 +126,16 @@ export function getExpiredDnsRecords() {
  },
  });
 }
+
+/**
+ * This Fn gets the base data that is needed for reconciliation
+ * but does that for all records in the db. Should be no more than 10k
+ *
+ * This serves as the ground truth for the entire DNS system, data is
+ * synchronized to Route53 from this
+ */
+export function getReconciliationData() {
+ return prisma.dnsRecord.findMany({
+ select: { username: true, subdomain: true, type: true, value: true },
+ });
+}

app/reconciler/DnsDbCompareStructureGenerator.server.ts

@@ -0,0 +1,42 @@
+import { get, set } from 'lodash';
+import { getReconciliationData } from '~/models/dns-record.server';
+import { buildDomain } from '../utils';
+
+import type { ReconcilerCompareStructure } from './ReconcilerTypes';
+
+class DnsDbCompareStructureGenerator {
+ #MUTATEDcompareStructure: ReconcilerCompareStructure = {};
+
+ /**
+ * This Fn reads the database `Record` table into a `ReconcilerCompareStructure` type object
+ * to be used for later comparison with Route53
+ */
+ generate = async (): Promise<ReconcilerCompareStructure> => {
+ const dnsData = await getReconciliationData();
+
+ // Populate compareStructure with data from the `Record` table
+ dnsData.forEach(({ username, subdomain, type, value }) => {
+ const fqdn = `${buildDomain(username, subdomain)}.`;
+
+ /**
+ * Get the previous value for fqdn.recordType (or an empty array if missing)
+ * and combine it with the new value we are just processing
+ *
+ * creates the following (example) structure
+ * {
+ * 'web.john.starchart.com.': {
+ * 'A': ['1.2.3.4']
+ * }
+ * }
+ */
+ const combinedValue = [...get(this.#MUTATEDcompareStructure, [fqdn, type], []), value];
+
+ // and set that value on our compare structure
+ set(this.#MUTATEDcompareStructure, [fqdn, type], combinedValue);
+ });
+
+ return this.#MUTATEDcompareStructure;
+ };
+}
+
+export default DnsDbCompareStructureGenerator;

app/reconciler/ReconcilerTypes.ts

@@ -0,0 +1,8 @@
+import type { DnsRecordType } from '@prisma/client';
+
+export interface ReconcilerCompareStructure {
+ [fqdn: string]: {
+ // Next layer is the record type
+ [recordType in DnsRecordType]?: string[];
+ };
+}

app/reconciler/Route53CompareStructureGenerator.server.ts

@@ -0,0 +1,64 @@
+import { set } from 'lodash';
+import { getDnsRecordSetPage } from '~/lib/dns.server';
+
+// Using this in JS code later, cannot `import type`
+import { DnsRecordType } from '@prisma/client';
+import type { ReconcilerCompareStructure } from './ReconcilerTypes';
+import type { ResourceRecordSet, ListResourceRecordSetsResponse } from '@aws-sdk/client-route-53';
+
+class Route53CompareStructureGenerator {
+ #MUTATEDcompareStructure: ReconcilerCompareStructure = {};
+
+ #processRecordSetPage = (recordSetPage: ResourceRecordSet[]) => {
+ recordSetPage.forEach((recordSet) => {
+ // Unsure as to how those could be undefined, but according to AWS sdk, they could
+ if (!recordSet.Name || !recordSet.Type) {
+ return;
+ }
+
+ // We only care about record types that we handle. NS, SOA, etc. should be ignored
+ // RecordSet.Type is given as `string`, so we have to do this for TS to compare them
+ if (!Object.values(DnsRecordType).includes(recordSet.Type as DnsRecordType)) {
+ return;
+ }
+
+ const value = recordSet.ResourceRecords?.map(({ Value }) => Value).filter(
+ (Value) => !!Value
+ ) as string[];
+
+ // and set that value on our compare structure
+ set(this.#MUTATEDcompareStructure, [recordSet.Name, recordSet.Type], value);
+ });
+ };
+
+ /**
+ * This Fn reads the complete Route53 zone into a `ReconcilerCompareStructure` type object
+ * to be used for later comparison with our database data
+ */
+ generate = async (): Promise<ReconcilerCompareStructure> => {
+ let morePages: boolean = true;
+ let nextFqdn: string | undefined = undefined;
+ let nextType: string | undefined = undefined;
+
+ while (morePages) {
+ const response: ListResourceRecordSetsResponse = await getDnsRecordSetPage(
+ nextFqdn,
+ nextType
+ );
+ morePages = !!response.IsTruncated;
+ nextFqdn = response.NextRecordName;
+ nextType = response.NextRecordType;
+
+ if (!response.ResourceRecordSets) {
+ continue;
+ }
+
+ // !!! `MUTATEDcompareStructure` is passed by reference and is mutated by fn !!!
+ this.#processRecordSetPage(response.ResourceRecordSets);
+ }
+
+ return this.#MUTATEDcompareStructure;
+ };
+}
+
+export default Route53CompareStructureGenerator;

app/reconciler/createChangeSetFromCompareStructures.server.ts

@@ -0,0 +1,103 @@
+import { isEqual } from 'lodash';
+import logger from '~/lib/logger.server';
+
+import { DnsRecordType } from '@prisma/client';
+
+import type { Change } from '@aws-sdk/client-route-53';
+import type { ReconcilerCompareStructure } from './ReconcilerTypes';
+
+interface CompareStructures {
+ dbStructure: ReconcilerCompareStructure;
+ route53Structure: ReconcilerCompareStructure;
+}
+
+export const createRemovedChangeSetFromCompareStructures = ({
+ dbStructure,
+ route53Structure,
+}: CompareStructures) => {
+ const toChange: Change[] = [];
+
+ /**
+ * If something is present in route53 but is completely missing in db,
+ * then we have to delete it
+ */
+ Object.keys(route53Structure).forEach((fqdn) => {
+ Object.keys(route53Structure[fqdn]).forEach((type) => {
+ const route53Value = route53Structure[fqdn][type as DnsRecordType]!;
+ const dbValue = dbStructure[fqdn]?.[type as DnsRecordType];
+
+ if (dbValue) {
+ // Both of them have this
+ return;
+ }
+
+ toChange.push({
+ Action: 'DELETE',
+ ResourceRecordSet: {
+ MultiValueAnswer: route53Value.length > 1,
+ Name: fqdn,
+ Type: type,
+ ResourceRecords: route53Value.map((Value) => ({ Value })),
+ TTL: 60 * 5,
+ },
+ });
+ });
+ });
+
+ return toChange;
+};
+
+export const createUpsertedChangeSetFromCompareStructures = ({
+ dbStructure,
+ route53Structure,
+}: CompareStructures) => {
+ const toChange: Change[] = [];
+
+ /**
+ * Now loop through all the data from the DB. If the data in
+ * Route53 is not the same, we should upsert
+ */
+ Object.keys(dbStructure).forEach((fqdn) => {
+ Object.keys(dbStructure[fqdn]).forEach((type) => {
+ const dbValue = dbStructure[fqdn][type as DnsRecordType]!;
+ const route53Value = route53Structure[fqdn]?.[type as DnsRecordType];
+ // When comparing, we do not care about the order
+ if (isEqual(dbValue?.sort(), route53Value?.sort())) {
+ // Both of them are equal
+ return;
+ }
+
+ /**
+ * https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/resource-record-sets-values-multivalue.html
+ *
+ * According to the above, only A, AAAA, CAA, MX, NAPTR, PTR, SPF, SRV and TXT records can be multi-value
+ */
+
+ if (
+ dbValue.length > 1 &&
+ !([DnsRecordType.A, DnsRecordType.AAAA, DnsRecordType.TXT] as DnsRecordType[]).includes(
+ type as DnsRecordType
+ )
+ ) {
+ logger.error(
+ 'Error creating DNS changeset Only A, AAAA and TXT records can be multivalue. Ignoring records',
+ { fqdn, type }
+ );
+ return;
+ }
+
+ toChange.push({
+ Action: 'UPSERT',
+ ResourceRecordSet: {
+ MultiValueAnswer: dbValue.length > 1,
+ Name: fqdn,
+ Type: type,
+ ResourceRecords: dbValue.map((Value) => ({ Value })),
+ TTL: 60 * 5,
+ },
+ });
+ });
+ });
+
+ return toChange;
+};

app/reconciler/index.ts

@@ -0,0 +1,39 @@
+import { executeChangeSet } from '~/lib/dns.server';
+import logger from '~/lib/logger.server';
+import DnsDbCompareStructureGenerator from './DnsDbCompareStructureGenerator.server';
+import Route53CompareStructureGenerator from './Route53CompareStructureGenerator.server';
+import {
+ createRemovedChangeSetFromCompareStructures,
+ createUpsertedChangeSetFromCompareStructures,
+} from './createChangeSetFromCompareStructures.server';
+
+// S3 limit for a ChangeSet
+const CHANGE_SET_MAX_SIZE = 1000;
+
+export const reconcile = async () => {
+ const [dbStructure, route53Structure] = await Promise.all([
+ new DnsDbCompareStructureGenerator().generate(),
+ new Route53CompareStructureGenerator().generate(),
+ ]);
+
+ const changeSet = [
+ ...createRemovedChangeSetFromCompareStructures({ dbStructure, route53Structure }),
+ ...createUpsertedChangeSetFromCompareStructures({ dbStructure, route53Structure }),
+ ];
+
+ // Limiting the changeSet to 1000 items. AWS limit
+
+ if (!changeSet.length) {
+ logger.debug('Reconciler found no changes to be pushed');
+ return;
+ }
+
+ logger.debug(`Reconciler intends to push the following ${changeSet.length} changes`, {
+ changeSet,
+ });
+
+ await executeChangeSet(changeSet.slice(0, CHANGE_SET_MAX_SIZE));
+
+ // Returning the changeSet size we are executing
+ return Math.min(changeSet.length, CHANGE_SET_MAX_SIZE);
+};