Add rolling-file appender to core logging (#84735) (#85524)

* You need to start somewhere

* revert comment

* rename default strategy to numeric

* add some tests

* fix some tests

* update documentation

* update generated doc

* change applyBaseConfig to be async

* fix integ tests

* add integration tests

* some renames

* more tests

* more tests

* nits on README

* some self review

* doc nits

* self review

* use `escapeRegExp` from lodash

* address some review comments

* a few more nits

* extract `isDevCliParent` check outside of LoggingSystem.upgrade

* log errors from context

* add defaults for policy/strategy

docs/development/core/server/kibana-plugin-core-server.appenderconfigtype.md

@@ -8,5 +8,5 @@
 <b>Signature:</b>
 
 ```typescript
-export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig;
+export declare type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig | RollingFileAppenderConfig;
 ```

packages/kbn-logging/src/appenders.ts

@@ -35,5 +35,5 @@ export interface Appender {
  * @internal
  */
 export interface DisposableAppender extends Appender {
-  dispose: () => void;
+  dispose: () => void | Promise<void>;
 }

src/core/server/logging/README.md

@@ -5,6 +5,10 @@
 - [Layouts](#layouts)
   - [Pattern layout](#pattern-layout)
   - [JSON layout](#json-layout)
+- [Appenders](#appenders)
+  - [Rolling File Appender](#rolling-file-appender)
+    - [Triggering Policies](#triggering-policies)
+    - [Rolling strategies](#rolling-strategies)
 - [Configuration](#configuration)
 - [Usage](#usage)
 - [Logging config migration](#logging-config-migration)
@@ -127,6 +131,138 @@ Outputs the process ID.
 With `json` layout log messages will be formatted as JSON strings that include timestamp, log level, context, message 
 text and any other metadata that may be associated with the log message itself.
 
+## Appenders
+
+### Rolling File Appender
+
+Similar to Log4j's `RollingFileAppender`, this appender will log into a file, and rotate it following a rolling
+strategy when the configured policy triggers.
+
+#### Triggering Policies
+
+The triggering policy determines when a rolling should occur.
+
+There are currently two policies supported: `size-limit` and `time-interval`.
+
+##### SizeLimitTriggeringPolicy
+
+This policy will rotate the file when it reaches a predetermined size.
+
+```yaml
+logging:
+  appenders:
+    rolling-file:
+      kind: rolling-file
+      path: /var/logs/kibana.log
+      policy:
+        kind: size-limit
+        size: 50mb
+      strategy:
+        //... 
+      layout:
+        kind: pattern
+```
+
+The options are:
+
+- `size`
+
+the maximum size the log file should reach before a rollover should be performed.
+
+The default value is `100mb`
+
+##### TimeIntervalTriggeringPolicy
+
+This policy will rotate the file every given interval of time.
+
+```yaml
+logging:
+  appenders:
+    rolling-file:
+      kind: rolling-file
+      path: /var/logs/kibana.log
+      policy:
+        kind: time-interval
+        interval: 10s
+        modulate: true
+      strategy:
+        //... 
+      layout:
+        kind: pattern
+```
+
+The options are:
+
+- `interval`
+
+How often a rollover should occur.
+
+The default value is `24h`
+
+- `modulate`
+
+Whether the interval should be adjusted to cause the next rollover to occur on the interval boundary.
+
+For example, when true, if the interval is `4h` and the current hour is 3 am then the first rollover will occur at 4 am 
+and then next ones will occur at 8 am, noon, 4pm, etc.
+
+The default value is `true`.
+
+#### Rolling strategies
+
+The rolling strategy determines how the rollover should occur: both the naming of the rolled files,
+and their retention policy.
+
+There is currently one strategy supported: `numeric`.
+
+##### NumericRollingStrategy
+
+This strategy will suffix the file with a given pattern when rolling,
+and will retains a fixed amount of rolled files.
+
+```yaml
+logging:
+  appenders:
+    rolling-file:
+      kind: rolling-file
+      path: /var/logs/kibana.log
+      policy:
+        // ...
+      strategy:
+        kind: numeric
+        pattern: '-%i'
+        max: 2
+      layout:
+        kind: pattern
+```
+
+For example, with this configuration:
+
+- During the first rollover kibana.log is renamed to kibana-1.log. A new kibana.log file is created and starts
+  being written to.
+- During the second rollover kibana-1.log is renamed to kibana-2.log and kibana.log is renamed to kibana-1.log.
+  A new kibana.log file is created and starts being written to.
+- During the third and subsequent rollovers, kibana-2.log is deleted, kibana-1.log is renamed to kibana-2.log and
+  kibana.log is renamed to kibana-1.log. A new kibana.log file is created and starts being written to.
+
+The options are:
+
+- `pattern`
+
+The suffix to append to the file path when rolling. Must include `%i`, as this is the value
+that will be converted to the file index.
+
+for example, with `path: /var/logs/kibana.log` and `pattern: '-%i'`, the created rolling files
+will be `/var/logs/kibana-1.log`, `/var/logs/kibana-2.log`, and so on.
+
+The default value is `-%i`
+
+- `max`
+
+The maximum number of files to keep. Once this number is reached, oldest files will be deleted.
+
+The default value is `7`
+
 ## Configuration
 
 As any configuration in the platform, logging configuration is validated against the predefined schema and if there are

src/core/server/logging/appenders/appenders.test.ts

@@ -19,10 +19,12 @@
 
 import { mockCreateLayout } from './appenders.test.mocks';
 
+import { ByteSizeValue } from '@kbn/config-schema';
 import { LegacyAppender } from '../../legacy/logging/appenders/legacy_appender';
 import { Appenders } from './appenders';
 import { ConsoleAppender } from './console/console_appender';
 import { FileAppender } from './file/file_appender';
+import { RollingFileAppender } from './rolling_file/rolling_file_appender';
 
 beforeEach(() => {
   mockCreateLayout.mockReset();
@@ -83,4 +85,13 @@ test('`create()` creates correct appender.', () => {
   });
 
   expect(legacyAppender).toBeInstanceOf(LegacyAppender);
+
+  const rollingFileAppender = Appenders.create({
+    kind: 'rolling-file',
+    path: 'path',
+    layout: { highlight: true, kind: 'pattern', pattern: '' },
+    strategy: { kind: 'numeric', max: 5, pattern: '%i' },
+    policy: { kind: 'size-limit', size: ByteSizeValue.parse('15b') },
+  });
+  expect(rollingFileAppender).toBeInstanceOf(RollingFileAppender);
 });

src/core/server/logging/appenders/appenders.ts

@@ -28,6 +28,10 @@ import {
 import { Layouts } from '../layouts/layouts';
 import { ConsoleAppender, ConsoleAppenderConfig } from './console/console_appender';
 import { FileAppender, FileAppenderConfig } from './file/file_appender';
+import {
+  RollingFileAppender,
+  RollingFileAppenderConfig,
+} from './rolling_file/rolling_file_appender';
 
 /**
  * Config schema for validting the shape of the `appenders` key in in {@link LoggerContextConfigType} or
@@ -39,10 +43,15 @@ export const appendersSchema = schema.oneOf([
   ConsoleAppender.configSchema,
   FileAppender.configSchema,
   LegacyAppender.configSchema,
+  RollingFileAppender.configSchema,
 ]);
 
 /** @public */
-export type AppenderConfigType = ConsoleAppenderConfig | FileAppenderConfig | LegacyAppenderConfig;
+export type AppenderConfigType =
+  | ConsoleAppenderConfig
+  | FileAppenderConfig
+  | LegacyAppenderConfig
+  | RollingFileAppenderConfig;
 
 /** @internal */
 export class Appenders {
@@ -57,10 +66,10 @@ export class Appenders {
     switch (config.kind) {
       case 'console':
         return new ConsoleAppender(Layouts.create(config.layout));
-
       case 'file':
         return new FileAppender(Layouts.create(config.layout), config.path);
-
+      case 'rolling-file':
+        return new RollingFileAppender(config);
       case 'legacy-appender':
         return new LegacyAppender(config.legacyLoggingConfig);
 

src/core/server/logging/appenders/rolling_file/mocks.ts

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { PublicMethodsOf } from '@kbn/utility-types';
+import type { Layout } from '@kbn/logging';
+import type { RollingFileContext } from './rolling_file_context';
+import type { RollingFileManager } from './rolling_file_manager';
+import type { TriggeringPolicy } from './policies/policy';
+import type { RollingStrategy } from './strategies/strategy';
+
+const createContextMock = (filePath: string) => {
+  const mock: jest.Mocked<RollingFileContext> = {
+    currentFileSize: 0,
+    currentFileTime: 0,
+    filePath,
+    refreshFileInfo: jest.fn(),
+  };
+  return mock;
+};
+
+const createStrategyMock = () => {
+  const mock: jest.Mocked<RollingStrategy> = {
+    rollout: jest.fn(),
+  };
+  return mock;
+};
+
+const createPolicyMock = () => {
+  const mock: jest.Mocked<TriggeringPolicy> = {
+    isTriggeringEvent: jest.fn(),
+  };
+  return mock;
+};
+
+const createLayoutMock = () => {
+  const mock: jest.Mocked<Layout> = {
+    format: jest.fn(),
+  };
+  return mock;
+};
+
+const createFileManagerMock = () => {
+  const mock: jest.Mocked<PublicMethodsOf<RollingFileManager>> = {
+    write: jest.fn(),
+    closeStream: jest.fn(),
+  };
+  return mock;
+};
+
+export const rollingFileAppenderMocks = {
+  createContext: createContextMock,
+  createStrategy: createStrategyMock,
+  createPolicy: createPolicyMock,
+  createLayout: createLayoutMock,
+  createFileManager: createFileManagerMock,
+};

src/core/server/logging/appenders/rolling_file/policies/index.ts

@@ -0,0 +1,70 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { schema } from '@kbn/config-schema';
+import moment from 'moment-timezone';
+import { assertNever } from '@kbn/std';
+import { TriggeringPolicy } from './policy';
+import { RollingFileContext } from '../rolling_file_context';
+import {
+  sizeLimitTriggeringPolicyConfigSchema,
+  SizeLimitTriggeringPolicyConfig,
+  SizeLimitTriggeringPolicy,
+} from './size_limit';
+import {
+  TimeIntervalTriggeringPolicyConfig,
+  TimeIntervalTriggeringPolicy,
+  timeIntervalTriggeringPolicyConfigSchema,
+} from './time_interval';
+
+export { TriggeringPolicy } from './policy';
+
+/**
+ * Any of the existing policy's configuration
+ *
+ * See {@link SizeLimitTriggeringPolicyConfig} and {@link TimeIntervalTriggeringPolicyConfig}
+ */
+export type TriggeringPolicyConfig =
+  | SizeLimitTriggeringPolicyConfig
+  | TimeIntervalTriggeringPolicyConfig;
+
+const defaultPolicy: TimeIntervalTriggeringPolicyConfig = {
+  kind: 'time-interval',
+  interval: moment.duration(24, 'hour'),
+  modulate: true,
+};
+
+export const triggeringPolicyConfigSchema = schema.oneOf(
+  [sizeLimitTriggeringPolicyConfigSchema, timeIntervalTriggeringPolicyConfigSchema],
+  { defaultValue: defaultPolicy }
+);
+
+export const createTriggeringPolicy = (
+  config: TriggeringPolicyConfig,
+  context: RollingFileContext
+): TriggeringPolicy => {
+  switch (config.kind) {
+    case 'size-limit':
+      return new SizeLimitTriggeringPolicy(config, context);
+    case 'time-interval':
+      return new TimeIntervalTriggeringPolicy(config, context);
+    default:
+      return assertNever(config);
+  }
+};