fix: calculate size of packument based on research data

workspaces/arborist/lib/packument-cache.js

@@ -2,6 +2,10 @@ const { LRUCache } = require('lru-cache')
 const { getHeapStatistics } = require('node:v8')
 const { log } = require('proc-log')
 
+// This is an in-memory cache that Pacote uses for packuments.
+// Packuments are usually cached on disk. This allows for rapid re-requests
+// of the same packument to bypass disk reads. The tradeoff here is memory
+// usage for disk reads.
 class PackumentCache extends LRUCache {
  static #heapLimit = Math.floor(getHeapStatistics().heap_size_limit)
 
@@ -13,17 +17,32 @@ class PackumentCache extends LRUCache {
  }
 
  constructor ({
+ // How much of this.#heapLimit to take up
  heapFactor = 0.25,
+ // How much of this.#maxSize we allow any one packument to take up
+ // Anything over this is not cached
  maxEntryFactor = 0.5,
  sizeKey = '_contentLength',
  } = {}) {
  const maxSize = Math.floor(PackumentCache.#heapLimit * heapFactor)
- const maxEntrySize = Math.floor(maxSize * maxEntryFactor)
+ // Max entry has an absolute maximum of 1M. After that it is more difficult
+ // to calculate memory usage based on content length.
+ const maxEntrySize = Math.max(Math.floor(maxSize * maxEntryFactor), 1000000)
  super({
  maxSize,
  maxEntrySize,
- // Don't cache if we dont know the size
- sizeCalculation: (p) => p[sizeKey] || maxEntrySize + 1,
+ sizeCalculation: (p) => {
+ // Don't cache if we dont know the size
+ if (!p[sizeKey]) {
+ return maxEntrySize + 1
+ }
+ // content under 10k has a ratio of size to memory of 0.45
+ if (p[sizeKey] < 10000) {
+ return p[sizeKey] * 0.45
+ }
+ // content between 10k and 1M has a ratio of size to memory of 0.6
+ return p[sizeKey] * 0.6
+ },
  dispose: (v, k) => {
  this.#disposed.add(k)
  this.#log(k, 'dispose')