Skip to content

Commit

Permalink
translate: refactor, add noswhere provider
Browse files Browse the repository at this point in the history
Also includes tests.

Signed-off-by: Semisol <[email protected]>
Co-authored-by: William Casarin <[email protected]>
Signed-off-by: William Casarin <[email protected]>
  • Loading branch information
Semisol and jb55 committed Apr 25, 2024
1 parent 5d97b9d commit 6845e0a
Show file tree
Hide file tree
Showing 8 changed files with 218 additions and 96 deletions.
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,14 @@ The Damus API backend for Damus Purple and other functionality.
#### Essential

- `DB_PATH`: Path to the folder where to save mdb files.
- `DEEPL_KEY`: API key for DeepL translation service (Can be set to something bogus for local testing with mock translations)
- `TESTFLIGHT_URL`: URL for the TestFlight app (optional)

#### Translations

- `TRANSLATION_PROVIDER`: The translation provider to use, can be: `mock`, `deepl`, `noswhere`
- `DEEPL_KEY`: The DeepL key to use for DeepL translations if enabled.
- `NOSWHERE_KEY`: The Noswhere key to use for Noswhere translations if enabled.

#### Apple In-App Purchase (IAP)

- `ENABLE_IAP_PAYMENTS`: Set to `"true"` to enable Apple In-App Purchase payment endpoints.
Expand Down
9 changes: 4 additions & 5 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,13 @@
"damus-api": "./src/index.js"
},
"scripts": {
"test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 tap test/*.test.js",
"test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 TRANSLATION_PROVIDER=mock tap test/*.test.js",
"test-translate": "ALLOW_HTTP_AUTH=\"true\" TRANSLATION_PROVIDER=mock tap test/translate.test.js",
"test-noswhere": "ALLOW_HTTP_AUTH=\"true\" TRANSLATION_PROVIDER=noswhere tap test/translate.test.js",
"debug-test": "ALLOW_HTTP_AUTH=\"true\" DEEPL_KEY=123 tap test/*.test.js --timeout=2400",
"start": "node src/index.js",
"mock_deepl": "node test_utils/mock_deepl.js",
"start_with_mock": "DEEPL_KEY=123 DEEPL_URL=http://localhost:8990 ENABLE_HTTP_AUTH=\"true\" node src/index.js",
"dev": "npm run mock_deepl & npm run start_with_mock",
"dev-debug": "npm run mock_deepl & DEEPL_KEY=123 DEEPL_URL=http://localhost:8990 ENABLE_HTTP_AUTH=\"true\" node inspect src/index.js",
"dev": "TRANSLATION_PROVIDER=mock ENABLE_HTTP_AUTH=\"true\" node src/index.js",
"dev-debug": "TRANSLATION_PROVIDER=mock ENABLE_HTTP_AUTH=\"true\" node --inspect src/index.js",
"type-check": "tsc --checkJs --allowJs src/*.js --noEmit --skipLibCheck",
"type-check-path": "tsc --checkJs --allowJs --noEmit --skipLibCheck"
},
Expand Down
97 changes: 40 additions & 57 deletions src/translate.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,71 +2,52 @@
const util = require('./server_helpers')
const crypto = require('crypto')
const current_time = require('./utils').current_time
const SUPPORTED_TRANSLATION_PROVIDERS = new Set(["mock", "noswhere", "deepl"])
let translation_provider = null

const translate_sources = new Set(['BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
const translate_targets = new Set(['BG', 'CS', 'DA', 'DE', 'EL', 'EN', 'EN-GB', 'EN-US', 'ES', 'ET', 'FI', 'FR', 'HU', 'ID', 'IT', 'JA', 'KO', 'LT', 'LV', 'NB', 'NL', 'PL', 'PT', 'PT-BR', 'PT-PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'TR', 'UK', 'ZH'])
if (!process.env.TRANSLATION_PROVIDER) {
throw new Error("expected TRANSLATION_PROVIDER")
}

const DEEPL_KEY = process.env.DEEPL_KEY
const DEEPL_URL = process.env.DEEPL_URL || 'https://api.deepl.com/v2/translate'
if (!SUPPORTED_TRANSLATION_PROVIDERS.has(process.env.TRANSLATION_PROVIDER)) {
throw new Error("translation provider not supported")
}

if (!DEEPL_KEY)
throw new Error("expected DEEPL_KEY env var")
// this is safe, as the input value is restricted to known good ones.
translation_provider = new (require("./translate/" + process.env.TRANSLATION_PROVIDER + ".js"))()

async function validate_payload(payload) {
if (!payload.source)
return { ok: false, message: 'missing source' }
if (!payload.target)
return { ok: false, message: 'missing target' }
if (!payload.q)
return { ok: false, message: 'missing q' }
if (!translate_sources.has(payload.source))
return { ok: false, message: 'invalid translation source' }
if (!translate_targets.has(payload.target))
return { ok: false, message: 'invalid translation target' }
if (typeof payload.source !== "string")
return { ok: false, message: 'bad source' }
if (typeof payload.target !== "string")
return { ok: false, message: 'bad target' }
if (typeof payload.q !== "string")
return { ok: false, message: 'bad q' }
if (!translation_provider.canTranslate(payload.source, payload.target))
return { ok: false, message: 'invalid translation source/target' }

return { ok: true, message: 'valid' }
}

function hash_payload(payload) {
const hash = crypto.createHash('sha256')
hash.update(payload.q)
hash.update(payload.source)
hash.update(payload.target)
hash.update(payload.source.toUpperCase())
hash.update(payload.target.toUpperCase())
return hash.digest()
}

async function deepl_translate_text(payload) {
let resp = await fetch(DEEPL_URL, {
method: 'POST',
headers: {
'Authorization': `DeepL-Auth-Key ${DEEPL_KEY}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({
text: [payload.q],
source_lang: payload.source,
target_lang: payload.target,
})
})

let data = await resp.json()

if (data.translations && data.translations.length > 0) {
return data.translations[0].text;
}

return null
}

async function translate_payload(api, res, payload, trans_id) {
// we might already be translating this
const job = api.translation.queue[trans_id]
if (job) {
let text = await job
if (text === null)
return util.error_response(res, 'deepl translation error')

return util.json_response(res, { text })
try {
let result = await job
return util.json_response(res, { text: result.text })
} catch(e) {
console.error("translation error: %o", e)
return util.error_response(res, 'translation error')
}
}

// we might have it in the database already
Expand All @@ -78,26 +59,28 @@ async function translate_payload(api, res, payload, trans_id) {
return util.json_response(res, { text })
}

const new_job = deepl_translate_text(payload)
const new_job = translation_provider.translate(payload.source, payload.target, payload.q)
api.translation.queue[trans_id] = new_job

let text = await new_job
if (text === null) {
let result

try {
result = await new_job
} catch(e) {
console.error("translation error: %o", e)
return util.error_response(res, 'translation error')
} finally {
delete api.translation.queue[trans_id]
return util.error_response(res, 'deepl translation error')
}

// return results immediately
util.json_response(res, { text })
util.json_response(res, { text: result.text })

// write result to db
await api.dbs.translations.put(trans_id, {
text: text,
text: result.text,
translated_at: current_time(),
payload: payload
})

delete api.translation.queue[trans_id]
}

function payload_is_data(q) {
Expand All @@ -111,8 +94,8 @@ function payload_is_data(q) {
async function handle_translate(api, req, res) {
let id
try {
const source = req.query.source.toUpperCase()
const target = req.query.target.toUpperCase()
const source = req.query.source.toLowerCase()
const target = req.query.target.toLowerCase()
const q = req.query.q
if (payload_is_data(q))
return util.invalid_request(res, `payload is data`)
Expand Down
57 changes: 57 additions & 0 deletions src/translate/deepl.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
const translate_sources = new Set([
'bg', 'cs', 'da', 'de', 'el',
'en', 'es', 'et', 'fi', 'fr',
'hu', 'id', 'it', 'ja', 'ko',
'lt', 'lv', 'nb', 'nl', 'pl',
'pt', 'ro', 'ru', 'sk', 'sl',
'sv', 'tr', 'uk', 'zh'
])
const translate_targets = new Set([
'bg', 'cs', 'da', 'de',
'el', 'en', 'en-gb', 'en-us',
'es', 'et', 'fi', 'fr',
'hu', 'id', 'it', 'ja',
'ko', 'lt', 'lv', 'nb',
'nl', 'pl', 'pt', 'pt-br',
'pt-pt', 'ro', 'ru', 'sk',
'sl', 'sv', 'tr', 'uk',
'zh'
])

module.exports = class DeepLTranslator {
#deeplURL = process.env.DEEPL_URL || 'https://api.deepl.com/v2/translate'
#deeplKey = process.env.DEEPL_KEY
constructor() {
if (!this.#deeplKey)
throw new Error("expected DEEPL_KEY env var")
}
canTranslate(from_lang, to_lang) {
return translate_sources.has(from_lang) && translate_targets.has(to_lang)
}
async translate(from_lang, to_lang, text) {
let resp = await fetch(this.#deeplURL, {
method: 'POST',
headers: {
'Authorization': `DeepL-Auth-Key ${this.#deeplKey}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({
text: [text],
source_lang: from_lang.toUpperCase(),
target_lang: to_lang.toUpperCase(),
})
})

if (!resp.ok) throw new Error("error translating: API failed with " + resp.status + " " + resp.statusText)

let data = await resp.json()

if (data.translations && data.translations.length > 0) {
return {
text: data.translations[0].text
}
}

throw new Error("error translating: no response")
}
}
13 changes: 13 additions & 0 deletions src/translate/mock.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
module.exports = class MockTranslator {
constructor() {

}
canTranslate(from_lang, to_lang) {
return true
}
async translate(from_lang, to_lang, text) {
return {
text: "Mock translation"
}
}
}
61 changes: 61 additions & 0 deletions src/translate/noswhere.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
module.exports = class NoswhereTranslator {
#noswhereURL = process.env.NOSWHERE_URL || 'https://translate.api.noswhere.com/api'
#noswhereKey = process.env.NOSWHERE_KEY
#type = "default"
#fromLangs = new Set()
#toLangs = new Set()
constructor() {
if (!this.#noswhereKey)
throw new Error("expected NOSWHERE_KEY env var")
this.#loadTranslationLangs()
}
async #loadTranslationLangs() {
let resp = await fetch(this.#noswhereURL + "/langs", {
method: 'GET',
headers: {
'X-Noswhere-Key': this.#noswhereKey,
'Content-Type': 'application/json'
}
})
let data = await resp.json()
if (!resp.ok) {
throw new Error(`error getting translation langs: API failed with ${resp.status} ${data.error} (request: ${resp.headers.get("x-noswhere-request")})`)
}
if (!data[this.#type]) {
throw new Error(`type ${this.#type} not supported for translation`)
}
this.#fromLangs = new Set(data[this.#type].from)
this.#toLangs = new Set(data[this.#type].to)
}
canTranslate(from_lang, to_lang) {
if (this.#fromLangs.size === 0) return true // assume true until we get the list of languages
return this.#fromLangs.has(from_lang) && this.#toLangs.has(to_lang)
}
async translate(from_lang, to_lang, text) {
let resp = await fetch(this.#noswhereURL + "/translate", {
method: 'POST',
headers: {
'X-Noswhere-Key': this.#noswhereKey,
'Content-Type': 'application/json'
},
body: JSON.stringify({
text: text,
src_lang: from_lang,
dst_lang: to_lang,
})
})

let data = await resp.json()
if (!resp.ok) {
throw new Error(`error translating: API failed with ${resp.status} ${data.error} (request: ${resp.headers.get("x-noswhere-request")})`)
}

if (data.result) {
return {
text: data.result
}
}

throw new Error("error translating: no response")
}
}
Loading

0 comments on commit 6845e0a

Please sign in to comment.