[Fleet] Show pipelines and mappings editor for input packages

x-pack/plugins/fleet/cypress/e2e/package_policy_real.cy.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import {
+  ADD_INTEGRATION_POLICY_BTN,
+  CREATE_PACKAGE_POLICY_SAVE_BTN,
+  INTEGRATION_NAME_LINK,
+  POLICY_EDITOR,
+} from '../screens/integrations';
+import { EXISTING_HOSTS_TAB } from '../screens/fleet';
+import { CONFIRM_MODAL } from '../screens/navigation';
+
+const TEST_PACKAGE = 'input_package-1.0.0';
+const agentPolicyId = 'test-input-package-policy';
+const agentPolicyName = 'Test input package policy';
+const inputPackagePolicyName = 'input-package-policy';
+
+function editPackagePolicyandShowAdvanced() {
+  cy.visit(`/app/integrations/detail/${TEST_PACKAGE}/policies`);
+
+  cy.getBySel(INTEGRATION_NAME_LINK).contains(inputPackagePolicyName).click();
+
+  cy.get('button').contains('Change defaults').click();
+  cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click();
+}
+describe('Input package create and edit package policy', () => {
+  before(() => {
+    cy.task('installTestPackage', TEST_PACKAGE);
+
+    cy.request({
+      method: 'POST',
+      url: `/api/fleet/agent_policies`,
+      body: {
+        id: agentPolicyId,
+        name: agentPolicyName,
+        description: 'desc',
+        namespace: 'default',
+        monitoring_enabled: [],
+      },
+      headers: { 'kbn-xsrf': 'cypress' },
+    });
+  });
+  after(() => {
+    // delete agent policy
+    cy.request({
+      method: 'POST',
+      url: `/api/fleet/agent_policies/delete`,
+      headers: { 'kbn-xsrf': 'cypress' },
+      body: JSON.stringify({
+        agentPolicyId,
+      }),
+    });
+    cy.task('uninstallTestPackage', TEST_PACKAGE);
+  });
+  it('should successfully create a package policy', () => {
+    cy.visit(`/app/integrations/detail/${TEST_PACKAGE}/overview`);
+    cy.getBySel(ADD_INTEGRATION_POLICY_BTN).click();
+
+    cy.getBySel(POLICY_EDITOR.POLICY_NAME_INPUT).click().clear().type(inputPackagePolicyName);
+    cy.getBySel('multiTextInput-paths')
+      .find('[data-test-subj="multiTextInputRow-0"]')
+      .click()
+      .type('/var/log/test.log');
+
+    cy.getBySel('multiTextInput-tags')
+      .find('[data-test-subj="multiTextInputRow-0"]')
+      .click()
+      .type('tag1');
+
+    cy.getBySel(POLICY_EDITOR.DATASET_SELECT).click().type('testdataset');
+
+    cy.getBySel(EXISTING_HOSTS_TAB).click();
+
+    cy.getBySel(POLICY_EDITOR.AGENT_POLICY_SELECT).click().get(`#${agentPolicyId}`).click();
+    cy.wait(500); // wait for policy id to be set
+    cy.getBySel(CREATE_PACKAGE_POLICY_SAVE_BTN).click();
+
+    cy.getBySel(CONFIRM_MODAL.CANCEL_BUTTON).click();
+  });
+
+  it('should show pipelines editor with link to pipeline', () => {
+    editPackagePolicyandShowAdvanced();
+    cy.getBySel(POLICY_EDITOR.INSPECT_PIPELINES_BTN).click();
+    cy.getBySel(CONFIRM_MODAL.CONFIRM_BUTTON).click();
+    cy.get('body').should('not.contain', 'Pipeline not found');
+    cy.get('body').should('contain', '"managed_by": "fleet"');
+  });
+  it('should show mappings editor with link to create custom template', () => {
+    editPackagePolicyandShowAdvanced();
+    cy.getBySel(POLICY_EDITOR.EDIT_MAPPINGS_BTN).click();
+    cy.getBySel(CONFIRM_MODAL.CONFIRM_BUTTON).click();
+    cy.get('body').should('contain', 'logs-testdataset@custom');
+  });
+});

x-pack/plugins/fleet/cypress/plugins/index.ts

@@ -5,12 +5,42 @@
  * 2.0.
  */
 
+import { promisify } from 'util';
+
+import fs from 'fs';
+
+import fetch from 'node-fetch';
 import { createEsClientForTesting } from '@kbn/test';
 
 const plugin: Cypress.PluginConfig = (on, config) => {
   const client = createEsClientForTesting({
     esUrl: config.env.ELASTICSEARCH_URL,
   });
+
+  async function kibanaFetch(opts: {
+    method: string;
+    path: string;
+    body?: any;
+    contentType?: string;
+  }) {
+    const { method, path, body, contentType } = opts;
+    const Authorization = `Basic ${Buffer.from(
+      `elastic:${config.env.ELASTICSEARCH_PASSWORD}`
+    ).toString('base64')}`;
+
+    const url = `${config.env.KIBANA_URL}${path}`;
+    const res = await fetch(url, {
+      method,
+      headers: {
+        'kbn-xsrf': 'cypress',
+        'Content-Type': contentType || 'application/json',
+        Authorization,
+      },
+      ...(body ? { body } : {}),
+    });
+
+    return res.json();
+  }
   on('task', {
     async insertDoc({ index, doc, id }: { index: string; doc: any; id: string }) {
       return client.create({ id, document: doc, index, refresh: 'wait_for' });
@@ -37,6 +67,23 @@ const plugin: Cypress.PluginConfig = (on, config) => {
         conflicts: 'proceed',
       });
     },
+    async installTestPackage(packageName: string) {
+      const zipPath = require.resolve('../packages/' + packageName + '.zip');
+      const zipContent = await promisify(fs.readFile)(zipPath, 'base64');
+      return kibanaFetch({
+        method: 'POST',
+        path: '/api/fleet/epm/packages',
+        body: Buffer.from(zipContent, 'base64'),
+        contentType: 'application/zip',
+      });
+    },
+
+    async uninstallTestPackage(packageName: string) {
+      return kibanaFetch({
+        method: 'DELETE',
+        path: `/api/fleet/epm/packages/${packageName}`,
+      });
+    },
   });
 };
 

x-pack/plugins/fleet/cypress/screens/integrations.ts

@@ -35,6 +35,14 @@ export const SETTINGS = {
   UNINSTALL_ASSETS_BTN: 'uninstallAssetsButton',
 };
 
+export const POLICY_EDITOR = {
+  POLICY_NAME_INPUT: 'packagePolicyNameInput',
+  DATASET_SELECT: 'datasetComboBox',
+  AGENT_POLICY_SELECT: 'agentPolicySelect',
+  INSPECT_PIPELINES_BTN: 'datastreamInspectPipelineBtn',
+  EDIT_MAPPINGS_BTN: 'datastreamEditMappingsBtn',
+};
+
 export const INTEGRATION_POLICIES_UPGRADE_CHECKBOX = 'epmDetails.upgradePoliciesCheckbox';
 
 export const getIntegrationCard = (integration: string) => `integration-card:epr:${integration}`;

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_hooks.tsx

@@ -6,8 +6,9 @@
  */
 
 import { useRouteMatch } from 'react-router-dom';
+import { useQuery } from '@tanstack/react-query';
 
-import { useLink } from '../../../../hooks';
+import { sendRequestForRq, useLink } from '../../../../hooks';
 
 export function usePackagePolicyEditorPageUrl(dataStreamId?: string) {
   const {
@@ -27,3 +28,32 @@ export function usePackagePolicyEditorPageUrl(dataStreamId?: string) {
 
   return `${baseUrl}${dataStreamId ? `?datastreamId=${encodeURIComponent(dataStreamId)}` : ''}`;
 }
+
+export function useIndexTemplateExists(
+  templateName: string,
+  enabled: boolean
+): {
+  exists?: boolean;
+  isLoading: boolean;
+} {
+  const { data, isLoading } = useQuery(
+    ['indexTemplateExists', templateName],
+    () =>
+      sendRequestForRq({
+        path: `/api/index_management/index_templates/${templateName}`,
+        method: 'get',
+      }),
+    { enabled: enabled || !!templateName }
+  );
+
+  if (isLoading) {
+    return {
+      isLoading: true,
+    };
+  }
+
+  return {
+    exists: !!data,
+    isLoading: false,
+  };
+}

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_mappings.tsx

@@ -18,7 +18,11 @@ import { usePackagePolicyEditorPageUrl } from './datastream_hooks';
 
 export interface PackagePolicyEditorDatastreamMappingsProps {
   packageInfo: PackageInfo;
-  packageInputStream: { id?: string; data_stream: { dataset: string; type: string } };
+  packageInputStream: {
+    id?: string;
+    data_stream: { dataset: string; type: string };
+  };
+  customDataset?: string;
 }
 
 function useComponentTemplates(dataStream: { dataset: string; type: string }) {
@@ -35,8 +39,10 @@ function useComponentTemplates(dataStream: { dataset: string; type: string }) {
 
 export const PackagePolicyEditorDatastreamMappings: React.FunctionComponent<
   PackagePolicyEditorDatastreamMappingsProps
-> = ({ packageInputStream, packageInfo }) => {
-  const dataStream = packageInputStream.data_stream;
+> = ({ packageInputStream, packageInfo, customDataset }) => {
+  const dataStream = customDataset
+    ? { ...packageInputStream.data_stream, dataset: customDataset }
+    : packageInputStream.data_stream;
   const pageUrl = usePackagePolicyEditorPageUrl(packageInputStream.id);
 
   const { application, docLinks } = useStartServices();

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_pipelines.tsx

@@ -31,6 +31,7 @@ import { usePackagePolicyEditorPageUrl } from './datastream_hooks';
 export interface PackagePolicyEditorDatastreamPipelinesProps {
   packageInfo: PackageInfo;
   packageInputStream: { id?: string; data_stream: { dataset: string; type: string } };
+  customDataset?: string;
 }
 
 interface PipelineItem {
@@ -92,8 +93,10 @@ function useDatastreamIngestPipelines(
 
 export const PackagePolicyEditorDatastreamPipelines: React.FunctionComponent<
   PackagePolicyEditorDatastreamPipelinesProps
-> = ({ packageInputStream, packageInfo }) => {
-  const dataStream = packageInputStream.data_stream;
+> = ({ packageInputStream, packageInfo, customDataset }) => {
+  const dataStream = customDataset
+    ? { ...packageInputStream.data_stream, dataset: customDataset }
+    : packageInputStream.data_stream;
   const { application, share, docLinks } = useStartServices();
   const ingestPipelineLocator = share.url.locators.get('INGEST_PIPELINES_APP_LOCATOR');
 

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx

@@ -94,6 +94,7 @@ export const DatasetComboBox: React.FC<{
         })}
         isClearable={false}
         isDisabled={isDisabled}
+        data-test-subj="datasetComboBox"
       />
       {valueAsOption && valueAsOption.value.package !== pkgName && (
         <>

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/multi_text_input.tsx

@@ -25,6 +25,7 @@ interface Props {
   errors?: Array<{ message: string; index?: number }>;
   isInvalid?: boolean;
   isDisabled?: boolean;
+  'data-test-subj'?: string;
 }
 
 interface RowProps {
@@ -69,6 +70,7 @@ const Row: FunctionComponent<RowProps> = ({
           autoFocus={autoFocus}
           disabled={isDisabled}
           onBlur={onBlur}
+          data-test-subj={`multiTextInputRow-${index}`}
         />
       </EuiFlexItem>
       {showDeleteButton && (
@@ -99,6 +101,7 @@ export const MultiTextInput: FunctionComponent<Props> = ({
   isInvalid,
   isDisabled,
   errors,
+  'data-test-subj': dataTestSubj,
 }) => {
   const [autoFocus, setAutoFocus] = useState(false);
   const [rows, setRows] = useState(() => defaultValue(value));
@@ -139,7 +142,7 @@ export const MultiTextInput: FunctionComponent<Props> = ({
 
   return (
     <>
-      <EuiFlexGroup gutterSize="s" direction="column">
+      <EuiFlexGroup gutterSize="s" direction="column" data-test-subj={dataTestSubj}>
         {rows.map((row, idx) => (
           <EuiFlexItem key={idx}>
             <Row

x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_stream.tsx

@@ -25,7 +25,10 @@ import { DATASET_VAR_NAME } from '../../../../../../../../../common/constants';
 
 import { useConfig, useGetDataStreams } from '../../../../../../../../hooks';
 
-import { mapPackageReleaseToIntegrationCardRelease } from '../../../../../../../../../common/services';
+import {
+  getRegistryDataStreamAssetBaseName,
+  mapPackageReleaseToIntegrationCardRelease,
+} from '../../../../../../../../../common/services';
 import type { ExperimentalDataStreamFeature } from '../../../../../../../../../common/types/models/epm';
 
 import type {
@@ -41,6 +44,8 @@ import { isAdvancedVar, validationHasErrors } from '../../../services';
 import { PackagePolicyEditorDatastreamPipelines } from '../../datastream_pipelines';
 import { PackagePolicyEditorDatastreamMappings } from '../../datastream_mappings';
 
+import { useIndexTemplateExists } from '../../datastream_hooks';
+
 import { ExperimentDatastreamSettings } from './experimental_datastream_settings';
 import { PackagePolicyInputVarField } from './package_policy_input_var_field';
 import { useDataStreamId } from './hooks';
@@ -90,12 +95,25 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
       !!packagePolicyInputStream.id &&
       packagePolicyInputStream.id === defaultDataStreamId;
     const isPackagePolicyEdit = !!packagePolicyId;
-    const isInputOnlyPackage = packageInfo.type === 'input';
 
-    const hasDatasetVar = packageInputStream.vars?.some(
-      (varDef) => varDef.name === DATASET_VAR_NAME
-    );
-    const showPipelinesAndMappings = !isInputOnlyPackage && !hasDatasetVar;
+    const customDatasetVar = packagePolicyInputStream.vars?.[DATASET_VAR_NAME];
+    const customDatasetVarValue = customDatasetVar?.value?.dataset || customDatasetVar?.value;
+
+    const { exists: indexTemplateExists, isLoading: isLoadingIndexTemplate } =
+      useIndexTemplateExists(
+        getRegistryDataStreamAssetBaseName({
+          dataset: customDatasetVarValue,
+          type: packageInputStream.data_stream.type,
+        }),
+        isPackagePolicyEdit
+      );
+
+    // only show pipelines and mappings if the matching index template exists
+    // in the legacy case (e.g logs package pre 2.0.0) the index template will not exist
+    // because we allowed dataset to be customized but didnt create a matching index template
+    // for the new dataset.
+    const showPipelinesAndMappings = !isLoadingIndexTemplate && indexTemplateExists;
+
     useEffect(() => {
       if (isDefaultDatastream && containerRef.current) {
         containerRef.current.scrollIntoView();
@@ -249,6 +267,7 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                         iconType={isShowingAdvanced ? 'arrowDown' : 'arrowRight'}
                         onClick={() => setIsShowingAdvanced(!isShowingAdvanced)}
                         flush="left"
+                        data-test-subj={`advancedStreamOptionsToggle-${packagePolicyInputStream.id}`}
                       >
                         <FormattedMessage
                           id="xpack.fleet.createPackagePolicy.stepConfigure.toggleAdvancedOptionsButtonText"
@@ -302,19 +321,20 @@ export const PackagePolicyInputStreamConfig = memo<Props>(
                         </EuiFlexItem>
                       );
                     })}
-                    {/* Only show datastream pipelines and mappings on edit and not for input packages*/}
                     {isPackagePolicyEdit && showPipelinesAndMappings && (
                       <>
                         <EuiFlexItem>
                           <PackagePolicyEditorDatastreamPipelines
                             packageInputStream={packagePolicyInputStream}
                             packageInfo={packageInfo}
+                            customDataset={customDatasetVarValue}
                           />
                         </EuiFlexItem>
                         <EuiFlexItem>
                           <PackagePolicyEditorDatastreamMappings
                             packageInputStream={packagePolicyInputStream}
                             packageInfo={packageInfo}
+                            customDataset={customDatasetVarValue}
                           />
                         </EuiFlexItem>
                       </>