Skip to content

Commit

Permalink
[Ingest Manager] Lift up registry/{stream,extract} functions (#83239)
Browse files Browse the repository at this point in the history
## Summary

  * Move stream utility functions from `server/services/epm/registry/streams.ts` to `server/services/epm/streams.ts`
    * They're only used in registry at the moment but aren't specific to registry 
  * Move archive extraction functions from `server/services/epm/registry/extract.ts` to `server/services/epm/archive.ts`
    * The Registry isn't the only service/code which needs to extract packages. Continue consolidating archive-related code under archive vs registry
  • Loading branch information
John Schulz authored Nov 12, 2020
1 parent 4932dc5 commit 208e86e
Show file tree
Hide file tree
Showing 9 changed files with 19 additions and 20 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,8 @@

import tar from 'tar';
import yauzl from 'yauzl';
import { bufferToStream, streamToBuffer } from './streams';

export interface ArchiveEntry {
path: string;
buffer?: Buffer;
}
import { bufferToStream, streamToBuffer } from '../streams';
import { ArchiveEntry } from './index';

export async function untarBuffer(
buffer: Buffer,
Expand Down
8 changes: 7 additions & 1 deletion x-pack/plugins/fleet/server/services/epm/archive/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,16 @@ import {
setArchiveFilelist,
deleteArchiveFilelist,
} from './cache';
import { ArchiveEntry, getBufferExtractor } from '../registry/extract';
import { getBufferExtractor } from './extract';
import { parseAndVerifyArchiveEntries } from './validation';

export * from './cache';
export { untarBuffer, unzipBuffer, getBufferExtractor } from './extract';

export interface ArchiveEntry {
path: string;
buffer?: Buffer;
}

export async function getArchivePackage({
archiveBuffer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ import {
RegistryVarsEntry,
} from '../../../../common/types';
import { PackageInvalidArchiveError } from '../../../errors';
import { ArchiveEntry, pkgToPkgKey } from '../registry';
import { ArchiveEntry } from './index';
import { pkgToPkgKey } from '../registry';

const MANIFESTS: Record<string, Buffer> = {};
const MANIFEST_NAME = 'manifest.yml';
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ import {
ElasticsearchAssetType,
InstallablePackage,
} from '../../../../types';
import { ArchiveEntry } from '../../registry';
import { getAsset, getPathParts } from '../../archive';
import { ArchiveEntry, getAsset, getPathParts } from '../../archive';
import { CallESAsCurrentUser } from '../../../../types';
import { saveInstalledEsRefs } from '../../packages/install';
import { getInstallationObject } from '../../packages';
Expand Down
6 changes: 3 additions & 3 deletions x-pack/plugins/fleet/server/services/epm/packages/assets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import { InstallablePackage } from '../../../types';
import * as Registry from '../registry';
import { getArchiveFilelist, getAsset } from '../archive';
import { ArchiveEntry, getArchiveFilelist, getAsset } from '../archive';

// paths from RegistryPackage are routes to the assets on EPR
// e.g. `/package/nginx/1.2.0/data_stream/access/fields/fields.yml`
Expand Down Expand Up @@ -51,14 +51,14 @@ export async function getAssetsData(
packageInfo: InstallablePackage,
filter = (path: string): boolean => true,
datasetName?: string
): Promise<Registry.ArchiveEntry[]> {
): Promise<ArchiveEntry[]> {
// TODO: Needs to be called to fill the cache but should not be required

await Registry.ensureCachedArchiveInfo(packageInfo.name, packageInfo.version, 'registry');

// Gather all asset data
const assets = getAssets(packageInfo, filter, datasetName);
const entries: Registry.ArchiveEntry[] = assets.map((path) => {
const entries: ArchiveEntry[] = assets.map((path) => {
const buffer = getAsset(path);

return { path, buffer };
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@
*/

import { AssetParts } from '../../../types';
import { getPathParts } from '../archive';
import { getBufferExtractor, splitPkgKey } from './index';
import { untarBuffer, unzipBuffer } from './extract';
import { getBufferExtractor, getPathParts, untarBuffer, unzipBuffer } from '../archive';
import { splitPkgKey } from './index';

const testPaths = [
{
Expand Down
4 changes: 1 addition & 3 deletions x-pack/plugins/fleet/server/services/epm/registry/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,11 @@ import {
unpackArchiveToCache,
} from '../archive';
import { fetchUrl, getResponse, getResponseStream } from './requests';
import { streamToBuffer } from './streams';
import { streamToBuffer } from '../streams';
import { getRegistryUrl } from './registry_url';
import { appContextService } from '../..';
import { PackageNotFoundError, PackageCacheError } from '../../../errors';

export { ArchiveEntry, getBufferExtractor } from './extract';

export interface SearchParams {
category?: CategoryId;
experimental?: boolean;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

import fetch, { FetchError, Response, RequestInit } from 'node-fetch';
import pRetry from 'p-retry';
import { streamToString } from './streams';
import { streamToString } from '../streams';
import { appContextService } from '../../app_context';
import { RegistryError, RegistryConnectionError, RegistryResponseError } from '../../../errors';
import { getProxyAgent, getRegistryProxyUrl } from './proxy';
Expand Down

0 comments on commit 208e86e

Please sign in to comment.