Skip to content

Commit

Permalink
[Fleet][EPM] Revert es-storage-related commits (#85942)
Browse files Browse the repository at this point in the history
* Revert "[Fleet][EPM] Save installed package assets in ES (#83391)"

This reverts commit 81a340e.

* Revert 00c2e96

Co-authored-by: Kibana Machine <[email protected]>
  • Loading branch information
John Schulz and kibanamachine authored Dec 15, 2020
1 parent 609d607 commit ec4fbf2
Show file tree
Hide file tree
Showing 14 changed files with 22 additions and 290 deletions.
2 changes: 1 addition & 1 deletion x-pack/plugins/fleet/common/constants/epm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';
export const MAX_TIME_COMPLETE_INSTALL = 60000;
Expand Down
6 changes: 0 additions & 6 deletions x-pack/plugins/fleet/common/types/models/epm.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
// TODO: Update when https://github.com/elastic/kibana/issues/53021 is closed
import { SavedObject, SavedObjectAttributes, SavedObjectReference } from 'src/core/public';
import {
ASSETS_SAVED_OBJECT_TYPE,
agentAssetTypes,
dataTypes,
defaultPackages,
Expand Down Expand Up @@ -269,7 +268,6 @@ export type PackageInfo =
export interface Installation extends SavedObjectAttributes {
installed_kibana: KibanaAssetReference[];
installed_es: EsAssetReference[];
package_assets: PackageAssetReference[];
es_index_patterns: Record<string, string>;
name: string;
version: string;
Expand Down Expand Up @@ -299,10 +297,6 @@ export type EsAssetReference = Pick<SavedObjectReference, 'id'> & {
type: ElasticsearchAssetType;
};

export type PackageAssetReference = Pick<SavedObjectReference, 'id'> & {
type: typeof ASSETS_SAVED_OBJECT_TYPE;
};

export type RequiredPackage = typeof requiredPackages;

export type DefaultPackages = typeof defaultPackages;
Expand Down
1 change: 0 additions & 1 deletion x-pack/plugins/fleet/server/constants/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ export {
PACKAGE_POLICY_SAVED_OBJECT_TYPE,
OUTPUT_SAVED_OBJECT_TYPE,
PACKAGES_SAVED_OBJECT_TYPE,
ASSETS_SAVED_OBJECT_TYPE,
INDEX_PATTERN_SAVED_OBJECT_TYPE,
ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
Expand Down
58 changes: 19 additions & 39 deletions x-pack/plugins/fleet/server/routes/epm/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,12 @@ import {
removeInstallation,
getLimitedPackages,
getInstallationObject,
getInstallation,
} from '../../services/epm/packages';
import { defaultIngestErrorHandler, ingestErrorToResponseOptions } from '../../errors';
import { splitPkgKey } from '../../services/epm/registry';
import { licenseService } from '../../services';
import { getArchiveEntry } from '../../services/epm/archive/cache';
import { getAsset } from '../../services/epm/archive/storage';
import { bufferToStream } from '../../services/epm/streams';

export const getCategoriesHandler: RequestHandler<
undefined,
Expand Down Expand Up @@ -108,51 +107,32 @@ export const getFileHandler: RequestHandler<TypeOf<typeof GetFileRequestSchema.p
try {
const { pkgName, pkgVersion, filePath } = request.params;
const savedObjectsClient = context.core.savedObjects.client;
const installation = await getInstallation({ savedObjectsClient, pkgName });
const useLocalFile = pkgVersion === installation?.version;

if (useLocalFile) {
const assetPath = `${pkgName}-${pkgVersion}/${filePath}`;
const fileBuffer = getArchiveEntry(assetPath);
// only pull local installation if we don't have it cached
const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath }));

// error, if neither is available
if (!fileBuffer && !storedAsset) {
const savedObject = await getInstallationObject({ savedObjectsClient, pkgName });
const pkgInstallSource = savedObject?.attributes.install_source;
// TODO: when package storage is available, remove installSource check and check cache and storage, remove registry call
if (pkgInstallSource === 'upload' && pkgVersion === savedObject?.attributes.version) {
const headerContentType = mime.contentType(path.extname(filePath));
if (!headerContentType) {
return response.custom({
body: `installed package file not found: ${filePath}`,
statusCode: 404,
body: `unknown content type for file: ${filePath}`,
statusCode: 400,
});
}

// if storedAsset is not available, fileBuffer *must* be
// b/c we error if we don't have at least one, and storedAsset is the least likely
const { buffer, contentType } = storedAsset
? {
contentType: storedAsset.media_type,
buffer: storedAsset.data_utf8
? Buffer.from(storedAsset.data_utf8, 'utf8')
: Buffer.from(storedAsset.data_base64, 'base64'),
}
: {
contentType: mime.contentType(path.extname(assetPath)),
buffer: fileBuffer,
};

if (!contentType) {
const archiveFile = getArchiveEntry(`${pkgName}-${pkgVersion}/${filePath}`);
if (!archiveFile) {
return response.custom({
body: `unknown content type for file: ${filePath}`,
statusCode: 400,
body: `uploaded package file not found: ${filePath}`,
statusCode: 404,
});
}

const headers: ResponseHeaders = {
'cache-control': 'max-age=10, public',
'content-type': headerContentType,
};
return response.custom({
body: buffer,
body: bufferToStream(archiveFile),
statusCode: 200,
headers: {
'cache-control': 'max-age=10, public',
'content-type': contentType,
},
headers,
});
} else {
const registryResponse = await getFile(pkgName, pkgVersion, filePath);
Expand Down
27 changes: 0 additions & 27 deletions x-pack/plugins/fleet/server/saved_objects/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
AGENT_POLICY_SAVED_OBJECT_TYPE,
PACKAGE_POLICY_SAVED_OBJECT_TYPE,
PACKAGES_SAVED_OBJECT_TYPE,
ASSETS_SAVED_OBJECT_TYPE,
AGENT_SAVED_OBJECT_TYPE,
AGENT_EVENT_SAVED_OBJECT_TYPE,
AGENT_ACTION_SAVED_OBJECT_TYPE,
Expand Down Expand Up @@ -305,39 +304,13 @@ const getSavedObjectTypes = (
type: { type: 'keyword' },
},
},
package_assets: {
type: 'nested',
properties: {
id: { type: 'keyword' },
type: { type: 'keyword' },
},
},
install_started_at: { type: 'date' },
install_version: { type: 'keyword' },
install_status: { type: 'keyword' },
install_source: { type: 'keyword' },
},
},
},
[ASSETS_SAVED_OBJECT_TYPE]: {
name: ASSETS_SAVED_OBJECT_TYPE,
hidden: false,
namespaceType: 'agnostic',
management: {
importableAndExportable: false,
},
mappings: {
properties: {
package_name: { type: 'keyword' },
package_version: { type: 'keyword' },
install_source: { type: 'keyword' },
asset_path: { type: 'keyword' },
media_type: { type: 'keyword' },
data_utf8: { type: 'text', index: false },
data_base64: { type: 'binary' },
},
},
},
});

export function registerSavedObjects(
Expand Down
140 changes: 0 additions & 140 deletions x-pack/plugins/fleet/server/services/epm/archive/storage.ts

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,7 @@
*/

import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
import {
InstallablePackage,
InstallSource,
PackageAssetReference,
MAX_TIME_COMPLETE_INSTALL,
ASSETS_SAVED_OBJECT_TYPE,
} from '../../../../common';
import { InstallablePackage, InstallSource, MAX_TIME_COMPLETE_INSTALL } from '../../../../common';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import {
AssetReference,
Expand All @@ -30,7 +24,6 @@ import { deleteKibanaSavedObjectsAssets } from './remove';
import { installTransform } from '../elasticsearch/transform/install';
import { createInstallation, saveKibanaAssetsRefs, updateVersion } from './install';
import { installIlmForDataStream } from '../elasticsearch/datastream_ilm/install';
import { saveArchiveEntries } from '../archive/storage';
import { ConcurrentInstallOperationError } from '../../../errors';

// this is only exported for testing
Expand Down Expand Up @@ -195,26 +188,11 @@ export async function _installPackage({
if (installKibanaAssetsError) throw installKibanaAssetsError;
await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);

const packageAssetResults = await saveArchiveEntries({
savedObjectsClient,
paths,
packageInfo,
installSource,
});
const packageAssetRefs: PackageAssetReference[] = packageAssetResults.saved_objects.map(
(result) => ({
id: result.id,
type: ASSETS_SAVED_OBJECT_TYPE,
})
);

// update to newly installed version when all assets are successfully installed
if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);

await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
install_version: pkgVersion,
install_status: 'installed',
package_assets: packageAssetRefs,
});

return [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ const mockInstallation: SavedObject<Installation> = {
id: 'test-pkg',
installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
package_assets: [],
es_index_patterns: { pattern: 'pattern-name' },
name: 'test package',
version: '1.0.0',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ const mockInstallation: SavedObject<Installation> = {
id: 'test-pkg',
installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
package_assets: [],
es_index_patterns: { pattern: 'pattern-name' },
name: 'test packagek',
version: '1.0.0',
Expand All @@ -33,7 +32,6 @@ const mockInstallationUpdateFail: SavedObject<Installation> = {
id: 'test-pkg',
installed_kibana: [{ type: KibanaSavedObjectType.dashboard, id: 'dashboard-1' }],
installed_es: [{ type: ElasticsearchAssetType.ingestPipeline, id: 'pipeline' }],
package_assets: [],
es_index_patterns: { pattern: 'pattern-name' },
name: 'test packagek',
version: '1.0.0',
Expand Down
Loading

0 comments on commit ec4fbf2

Please sign in to comment.