425 lines
14 KiB
YAML
425 lines
14 KiB
YAML
# Copyright OpenSearch Contributors
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Default values for data-prepper.
|
|
# This is a YAML-formatted file.
|
|
# Declare variables to be passed into your templates.
|
|
|
|
replicaCount: 1
|
|
|
|
global:
|
|
# Set if you want to change the default docker registry, e.g. a private one.
|
|
dockerRegistry: ""
|
|
|
|
image:
|
|
# -- The image repository from which to pull the Data Prepper image
|
|
repository: opensearchproject/data-prepper
|
|
# -- The image tag to pull. Default: IfNotPresent
|
|
pullPolicy: IfNotPresent
|
|
# -- Overrides the image tag whose default is the chart appVersion.
|
|
tag: ""
|
|
|
|
# -- List of imagePullSecrets to use if the Docker image is stored in a private registry
|
|
imagePullSecrets: []
|
|
# -- Override the default name for the deployment
|
|
nameOverride: ""
|
|
# -- Override the default fullname for the deployment
|
|
fullnameOverride: ""
|
|
|
|
# -- Extra environment variables to pass to the Data Prepper container
|
|
extraEnvs:
|
|
# - name: "JAVA_OPTS"
|
|
# value: "-Dlog4j2.debug=true"
|
|
- name: "LOGGER_PASSWORD"
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: prepper-creds
|
|
key: logger
|
|
# Check https://opensearch.org/docs/latest/data-prepper/managing-data-prepper/configuring-data-prepper/
|
|
# for more information on the configuration options
|
|
# -- Data Prepper configuration
|
|
config:
|
|
# -- Main Data Prepper configuration file content
|
|
data-prepper-config.yaml: |
|
|
ssl: false
|
|
# circuit_breakers:
|
|
# heap:
|
|
# usage: 2gb
|
|
# reset: 30s
|
|
# check_interval: 5s
|
|
|
|
# -- Log4j2 configuration for Data Prepper logging
|
|
log4j2-rolling.properties: |
|
|
#
|
|
# Copyright OpenSearch Contributors
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
|
|
status = error
|
|
dest = err
|
|
name = PropertiesConfig
|
|
|
|
property.filename = log/data-prepper/data-prepper.log
|
|
|
|
appender.console.type = Console
|
|
appender.console.name = STDOUT
|
|
appender.console.layout.type = PatternLayout
|
|
appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n
|
|
|
|
appender.rolling.type = RollingFile
|
|
appender.rolling.name = RollingFile
|
|
appender.rolling.fileName = ${filename}
|
|
appender.rolling.filePattern = logs/data-prepper.log.%d{MM-dd-yy-HH}-%i.gz
|
|
appender.rolling.layout.type = PatternLayout
|
|
appender.rolling.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n
|
|
appender.rolling.policies.type = Policies
|
|
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
|
|
appender.rolling.policies.time.interval = 1
|
|
appender.rolling.policies.time.modulate = true
|
|
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
|
|
appender.rolling.policies.size.size=100MB
|
|
appender.rolling.strategy.type = DefaultRolloverStrategy
|
|
appender.rolling.strategy.max = 168
|
|
|
|
rootLogger.level = warn
|
|
rootLogger.appenderRef.stdout.ref = STDOUT
|
|
rootLogger.appenderRef.file.ref = RollingFile
|
|
|
|
logger.pipeline.name = org.opensearch.dataprepper.pipeline
|
|
logger.pipeline.level = info
|
|
|
|
logger.parser.name = org.opensearch.dataprepper.parser
|
|
logger.parser.level = info
|
|
|
|
logger.plugins.name = org.opensearch.dataprepper.plugins
|
|
logger.plugins.level = info
|
|
|
|
# For OpenSearch Data Prepper is crucial for defining the behavior and structure of your data processing pipelines.
|
|
# Each pipeline is defined with a unique name and can include `source`, `processor`, and `sink` components to ingest,
|
|
# process, and output data respectively. This flexible configuration allows for the creation of complex data processing
|
|
# flows, including the routing of data between pipelines.
|
|
# For detailed information on the available options and to get the most up-to-date guidance on configuring `pipeline.yaml`,
|
|
# please consult the [OpenSearch Documentation on Pipelines](https://opensearch.org/docs/2.4/data-prepper/pipelines/pipelines/).
|
|
# This resource provides comprehensive examples and explanations of each component, ensuring you can tailor your Data Prepper
|
|
# deployment to meet your specific data processing needs.
|
|
|
|
# -- Pipeline configuration
|
|
pipelineConfig:
|
|
# If 'true', a secret containing a demo pipeline configuration with random source and stdout sink will be created.
|
|
# If left undefined, the demo pipeline will be used only when no other pipeline is configured below
|
|
demoPipeline: ""
|
|
# -- The name of the existing secret containing the pipeline configuration.
|
|
# If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`.
|
|
existingSecret: ""
|
|
# If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below.
|
|
enabled: true
|
|
# The configuration of the pipeline see https://opensearch.org/docs/2.4/data-prepper/pipelines/pipelines/
|
|
config:
|
|
## Provide your pipeline configuration here if 'enabled' is set to true. See documentation for more advanced pipelines
|
|
# simple-sample-pipeline:
|
|
# workers: 2 # the number of workers
|
|
# delay: 5000 # in milliseconds, how long workers wait between read attempt
|
|
cpanel-log-router:
|
|
workers: 2
|
|
delay: 100
|
|
source:
|
|
http:
|
|
path: /logs
|
|
ssl: false
|
|
port: 2021
|
|
|
|
buffer:
|
|
bounded_blocking:
|
|
buffer_size: 2048
|
|
batch_size: 128
|
|
|
|
route:
|
|
- systemLog : '/log_source == "system.syslog"'
|
|
- accessLog : '/log_source == "cpanel.access_log"'
|
|
- auditLog : '/log_source == "audit.audit.log"'
|
|
- kernelLog: '/log_source == "system.kern.log"'
|
|
|
|
sink:
|
|
- pipeline:
|
|
name: cpanel-access-pipeline
|
|
routes:
|
|
- accessLog
|
|
|
|
- pipeline:
|
|
name: cpanel-system-syslog-pipeline
|
|
routes:
|
|
- systemLog
|
|
|
|
- pipeline:
|
|
name: cpanel-audit-pipeline
|
|
|
|
- pipeline:
|
|
name: cpanel-kernel-pipeline
|
|
|
|
- pipeline:
|
|
name: cpanel-all-log
|
|
|
|
|
|
|
|
cpanel-access-pipeline:
|
|
source:
|
|
pipeline:
|
|
name: cpanel-log-router
|
|
|
|
processor:
|
|
- grok:
|
|
break_on_match: false
|
|
match:
|
|
log:
|
|
- '^%{IP:client_ip} %{NOTSPACE:identity} %{NOTSPACE:user} \[%{GREEDYDATA:timestamp}\] "%{WORD:method} %{URIPATHPARAM:url} HTTP/%{NUMBER:http_version}" %{INT:http_status:int} %{NONNEGINT:bytes:long} "%{NOTSPACE:referrer}" "%{GREEDYDATA:user_agent}" "%{NOTSPACE:auth_method}" "%{GREEDYDATA:x_forwarded_for}" %{POSINT:service_port:int}$'
|
|
target_key: parsed
|
|
|
|
- date:
|
|
match:
|
|
- key: /parsed/timestamp
|
|
patterns:
|
|
- "MM/dd/yyyy:HH:mm:ss Z"
|
|
destination: "@timestamp"
|
|
|
|
sink:
|
|
- opensearch:
|
|
hosts:
|
|
- "https://brilliant-opensearch-cluster-master:9200"
|
|
insecure: false
|
|
username: logger
|
|
password: "Enp129s0f0-l0gin"
|
|
index_type: custom
|
|
cert: /usr/share/dataprepper/cert/root-ca.pem
|
|
index: cpanel-access-log-%{yyyy.MM.dd}
|
|
|
|
cpanel-system-syslog-pipeline:
|
|
source:
|
|
pipeline:
|
|
name: cpanel-log-router
|
|
processor:
|
|
- drop_events:
|
|
drop_when: '/log_source != "system.syslog"'
|
|
- grok:
|
|
break_on_match: false
|
|
match:
|
|
log:
|
|
- '^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:hostname} %{DATA:process}\[(?:%{POSINT:pid})\]: %{GREEDYDATA:message}$'
|
|
target_key: parsed
|
|
sink:
|
|
- opensearch:
|
|
hosts:
|
|
- "https://brilliant-opensearch-cluster-master:9200"
|
|
insecure: false
|
|
username: logger
|
|
password: "Enp129s0f0-l0gin"
|
|
index_type: custom
|
|
cert: /usr/share/dataprepper/cert/root-ca.pem
|
|
index: cpanel-system-syslog-%{yyyy.MM.dd}
|
|
|
|
cpanel-audit-pipeline:
|
|
source:
|
|
pipeline:
|
|
name: cpanel-log-router
|
|
processor:
|
|
- drop_events:
|
|
drop_when: '/log_source != "audit.audit.log"'
|
|
|
|
- grok:
|
|
break_on_match: true
|
|
match:
|
|
log:
|
|
- '^type=%{WORD:log_type} msg=audit\(%{INT:timestamp_seconds}\.%{INT:timestamp_fraction}:%{INT:event_id}\): %{GREEDYDATA:msg}'
|
|
target_key: audit
|
|
|
|
- grok:
|
|
break_on_match: true
|
|
grok_when: '/audit/log_type == "SYSCALL"'
|
|
match:
|
|
/audit/msg:
|
|
- ^arch=%{WORD:arch} syscall=%{INT:syscall_id} success=%{WORD:success}
|
|
exit=%{INT:exit_code} a0=%{NOTSPACE:a0} a1=%{NOTSPACE:a1} a2=%{NOTSPACE:a2}
|
|
a3=%{NOTSPACE:a3} items=%{INT:items} ppid=%{INT:ppid} pid=%{INT:pid} auid=%{INT:auid}
|
|
uid=%{INT:uid} gid=%{INT:gid} euid=%{INT:euid} suid=%{INT:suid} fsuid=%{INT:fsuid}
|
|
egid=%{INT:egid} sgid=%{INT:sgid} fsgid=%{INT:fsgid} tty=%{GREEDYDATA:terminal}
|
|
ses=%{INT:ses} comm=\"%{NOTSPACE:comm}\" exe=\"%{GREEDYDATA:exe}\" subj=%{GREEDYDATA:subject}
|
|
key=\"%{WORD:key}\".*ARCH=%{GREEDYDATA:architecture} SYSCALL=%{WORD:syscall} AUID=\"%{GREEDYDATA:AUID}\"
|
|
UID=\"%{GREEDYDATA:UID}\" GID=\"%{GREEDYDATA:GID}\" EUID=\"%{GREEDYDATA:EUID}\" SUID=\"%{GREEDYDATA:SUID}\"
|
|
FSUID=\"%{GREEDYDATA:FSUID}\"
|
|
EGID=\"%{GREEDYDATA:EGID}\" SGID=\"%{GREEDYDATA:SGID}\" FSGID=\"%{GREEDYDATA:FSGID}\"
|
|
target_key: syscall
|
|
- date:
|
|
match:
|
|
- key: /audit/timestamp_seconds
|
|
patterns:
|
|
- epoch_second
|
|
destination: "@timestamp"
|
|
|
|
|
|
sink:
|
|
- opensearch:
|
|
hosts:
|
|
- "https://brilliant-opensearch-cluster-master:9200"
|
|
insecure: false
|
|
username: logger
|
|
password: "Enp129s0f0-l0gin"
|
|
index_type: custom
|
|
cert: /usr/share/dataprepper/cert/root-ca.pem
|
|
index: cpanel-audit-log-%{yyyy.MM.dd}
|
|
|
|
|
|
cpanel-kernel-pipeline:
|
|
source:
|
|
pipeline:
|
|
name: cpanel-log-router
|
|
processor:
|
|
- drop_events:
|
|
drop_when: '/log_source != "system.kern.log"'
|
|
|
|
sink:
|
|
- opensearch:
|
|
hosts:
|
|
- "https://brilliant-opensearch-cluster-master:9200"
|
|
insecure: false
|
|
username: logger
|
|
password: "Enp129s0f0-l0gin"
|
|
index_type: custom
|
|
cert: /usr/share/dataprepper/cert/root-ca.pem
|
|
index: cpanel-kernel-log-%{yyyy.MM.dd}
|
|
|
|
|
|
cpanel-all-log:
|
|
source:
|
|
pipeline:
|
|
name: cpanel-log-router
|
|
processor:
|
|
- drop_events:
|
|
drop_when: '/log_source == "audit.audit.log"'
|
|
- drop_events:
|
|
drop_when: '/log_source == "system.syslog"'
|
|
- drop_events:
|
|
drop_when: '/log_source == "cpanel.access_log"'
|
|
- drop_events:
|
|
drop_when: '/log_source == "system.kern.log"'
|
|
|
|
sink:
|
|
- opensearch:
|
|
hosts:
|
|
- "https://brilliant-opensearch-cluster-master:9200"
|
|
insecure: false
|
|
username: logger
|
|
password: "Enp129s0f0-l0gin"
|
|
index_type: custom
|
|
cert: /usr/share/dataprepper/cert/root-ca.pem
|
|
index: cpanel-all-log-%{yyyy.MM.dd}
|
|
|
|
|
|
# -- Data Prepper ports
|
|
ports:
|
|
# -- The port that the source is running on. Default value is 2021. Valid options are between 0 and 65535.
|
|
# https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/http-source/
|
|
- name: http-source
|
|
port: 2021
|
|
# # -- The port that the otel_trace_source source runs on. Default value is 21890.
|
|
# # https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-trace-source/
|
|
# - name: otel-traces
|
|
# port: 21890
|
|
# # -- The port that the OpenTelemtry metrics source runs on. Default value is 21891.
|
|
# # https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-metrics-source/
|
|
# - name: otel-metrics
|
|
# port: 21891
|
|
# # -- Represents the port that the otel_logs_source source is running on. Default value is 21892.
|
|
# # https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-logs-source/
|
|
# - name: otel-logs
|
|
# port: 21892
|
|
|
|
serviceAccount:
|
|
# -- Specifies whether a service account should be created
|
|
create: true
|
|
# -- Automatically mount a ServiceAccount's API credentials?
|
|
automount: true
|
|
# -- Annotations to add to the service account
|
|
annotations: {}
|
|
# -- The name of the service account to use.
|
|
# If not set and create is true, a name is generated using the fullname template
|
|
name: ""
|
|
|
|
podAnnotations: {}
|
|
podLabels: {}
|
|
|
|
podSecurityContext: {}
|
|
# fsGroup: 2000
|
|
|
|
securityContext: {}
|
|
# capabilities:
|
|
# drop:
|
|
# - ALL
|
|
# readOnlyRootFilesystem: true
|
|
# runAsNonRoot: true
|
|
# runAsUser: 1000
|
|
|
|
initContainers: []
|
|
|
|
service:
|
|
type: ClusterIP
|
|
port: 2021
|
|
|
|
ingress:
|
|
enabled: false
|
|
className: "nginx"
|
|
annotations:
|
|
kubernetes.io/ingress.class: nginx
|
|
# kubernetes.io/tls-acme: "true"
|
|
hosts:
|
|
- paths:
|
|
- path: /logs
|
|
pathType: Prefix
|
|
tls: []
|
|
# - secretName: chart-example-tls
|
|
# hosts:
|
|
# - chart-example.local
|
|
|
|
resources:
|
|
# We usually recommend not to specify default resources and to leave this as a conscious
|
|
# choice for the user. This also increases chances charts run on environments with little
|
|
# resources, such as Minikube. If you do want to specify resources, uncomment the following
|
|
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
|
|
limits:
|
|
cpu: 4
|
|
memory: 6Gi
|
|
requests:
|
|
cpu: 2
|
|
memory: 4Gi
|
|
|
|
autoscaling:
|
|
# done for testing, in production this will be true
|
|
enabled: false
|
|
minReplicas: 1
|
|
maxReplicas: 100
|
|
targetCPUUtilizationPercentage: 75
|
|
targetMemoryUtilizationPercentage: 75
|
|
|
|
# Additional volumes on the output Deployment definition.
|
|
volumes:
|
|
- name: root-ca
|
|
secret:
|
|
secretName: opensearch-dashboard-ca
|
|
# - name: foo
|
|
# secret:
|
|
# secretName: mysecret
|
|
# optional: false
|
|
|
|
# Additional volumeMounts on the output Deployment definition.
|
|
volumeMounts:
|
|
- name: root-ca
|
|
readOnly: true
|
|
mountPath: /usr/share/dataprepper/cert
|
|
# - name: foo
|
|
# mountPath: "/etc/foo"
|
|
# readOnly: true
|
|
|
|
nodeSelector: {}
|
|
|
|
tolerations: []
|
|
|
|
affinity: {}
|