Convert helm-charts-observability to regular directory
This commit is contained in:
Mosiur Rahman
360
helm-charts-observability/charts/data-prepper/values.yaml
Normal file
360
helm-charts-observability/charts/data-prepper/values.yaml
Normal file
@@ -0,0 +1,360 @@
|
||||
# Copyright OpenSearch Contributors
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Default values for data-prepper.
|
||||
# This is a YAML-formatted file.
|
||||
# Declare variables to be passed into your templates.
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
global:
|
||||
# Set if you want to change the default docker registry, e.g. a private one.
|
||||
dockerRegistry: ""
|
||||
|
||||
image:
|
||||
# -- The image repository from which to pull the Data Prepper image
|
||||
repository: opensearchproject/data-prepper
|
||||
# -- The image tag to pull. Default: IfNotPresent
|
||||
pullPolicy: IfNotPresent
|
||||
# -- Overrides the image tag whose default is the chart appVersion.
|
||||
tag: ""
|
||||
|
||||
# -- List of imagePullSecrets to use if the Docker image is stored in a private registry
|
||||
imagePullSecrets: []
|
||||
# -- Override the default name for the deployment
|
||||
nameOverride: ""
|
||||
# -- Override the default fullname for the deployment
|
||||
fullnameOverride: ""
|
||||
|
||||
# -- Extra environment variables to pass to the Data Prepper container
|
||||
extraEnvs:
|
||||
# - name: "JAVA_OPTS"
|
||||
# value: "-Dlog4j2.debug=true"
|
||||
- name: "LOGGER_PASSWORD"
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: prepper-creds
|
||||
key: logger
|
||||
# Check https://opensearch.org/docs/latest/data-prepper/managing-data-prepper/configuring-data-prepper/
|
||||
# for more information on the configuration options
|
||||
# -- Data Prepper configuration
|
||||
config:
|
||||
# -- Main Data Prepper configuration file content
|
||||
data-prepper-config.yaml: |
|
||||
ssl: false
|
||||
# circuit_breakers:
|
||||
# heap:
|
||||
# usage: 2gb
|
||||
# reset: 30s
|
||||
# check_interval: 5s
|
||||
|
||||
# -- Log4j2 configuration for Data Prepper logging
|
||||
log4j2-rolling.properties: |
|
||||
#
|
||||
# Copyright OpenSearch Contributors
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
|
||||
status = error
|
||||
dest = err
|
||||
name = PropertiesConfig
|
||||
|
||||
property.filename = log/data-prepper/data-prepper.log
|
||||
|
||||
appender.console.type = Console
|
||||
appender.console.name = STDOUT
|
||||
appender.console.layout.type = PatternLayout
|
||||
appender.console.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n
|
||||
|
||||
appender.rolling.type = RollingFile
|
||||
appender.rolling.name = RollingFile
|
||||
appender.rolling.fileName = ${filename}
|
||||
appender.rolling.filePattern = logs/data-prepper.log.%d{MM-dd-yy-HH}-%i.gz
|
||||
appender.rolling.layout.type = PatternLayout
|
||||
appender.rolling.layout.pattern = %d{ISO8601} [%t] %-5p %40C - %m%n
|
||||
appender.rolling.policies.type = Policies
|
||||
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
|
||||
appender.rolling.policies.time.interval = 1
|
||||
appender.rolling.policies.time.modulate = true
|
||||
appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
|
||||
appender.rolling.policies.size.size=100MB
|
||||
appender.rolling.strategy.type = DefaultRolloverStrategy
|
||||
appender.rolling.strategy.max = 168
|
||||
|
||||
rootLogger.level = warn
|
||||
rootLogger.appenderRef.stdout.ref = STDOUT
|
||||
rootLogger.appenderRef.file.ref = RollingFile
|
||||
|
||||
logger.pipeline.name = org.opensearch.dataprepper.pipeline
|
||||
logger.pipeline.level = info
|
||||
|
||||
logger.parser.name = org.opensearch.dataprepper.parser
|
||||
logger.parser.level = info
|
||||
|
||||
logger.plugins.name = org.opensearch.dataprepper.plugins
|
||||
logger.plugins.level = info
|
||||
|
||||
# For OpenSearch Data Prepper is crucial for defining the behavior and structure of your data processing pipelines.
|
||||
# Each pipeline is defined with a unique name and can include `source`, `processor`, and `sink` components to ingest,
|
||||
# process, and output data respectively. This flexible configuration allows for the creation of complex data processing
|
||||
# flows, including the routing of data between pipelines.
|
||||
# For detailed information on the available options and to get the most up-to-date guidance on configuring `pipeline.yaml`,
|
||||
# please consult the [OpenSearch Documentation on Pipelines](https://opensearch.org/docs/2.4/data-prepper/pipelines/pipelines/).
|
||||
# This resource provides comprehensive examples and explanations of each component, ensuring you can tailor your Data Prepper
|
||||
# deployment to meet your specific data processing needs.
|
||||
|
||||
# -- Pipeline configuration
|
||||
pipelineConfig:
|
||||
# If 'true', a secret containing a demo pipeline configuration with random source and stdout sink will be created.
|
||||
# If left undefined, the demo pipeline will be used only when no other pipeline is configured below
|
||||
demoPipeline: ""
|
||||
# -- The name of the existing secret containing the pipeline configuration.
|
||||
# If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`.
|
||||
existingSecret: ""
|
||||
# If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below.
|
||||
enabled: true
|
||||
# The configuration of the pipeline see https://opensearch.org/docs/2.4/data-prepper/pipelines/pipelines/
|
||||
config:
|
||||
## Provide your pipeline configuration here if 'enabled' is set to true. See documentation for more advanced pipelines
|
||||
# simple-sample-pipeline:
|
||||
# workers: 2 # the number of workers
|
||||
# delay: 5000 # in milliseconds, how long workers wait between read attempt
|
||||
cpanel-log-router:
|
||||
workers: 2
|
||||
delay: 100
|
||||
source:
|
||||
http:
|
||||
path: /logs
|
||||
ssl: false
|
||||
port: 2021
|
||||
|
||||
buffer:
|
||||
bounded_blocking:
|
||||
buffer_size: 2048
|
||||
batch_size: 128
|
||||
|
||||
route:
|
||||
- systemLog : '/log_source == "system.syslog"'
|
||||
- accessLog : '/log_source == "cpanel.access_log"'
|
||||
- auditLog : '/log_source == "audit.audit.log"'
|
||||
|
||||
sink:
|
||||
- pipeline:
|
||||
name: cpanel-access-pipeline
|
||||
routes:
|
||||
- accessLog
|
||||
|
||||
- pipeline:
|
||||
name: cpanel-system-syslog-pipeline
|
||||
routes:
|
||||
- systemLog
|
||||
|
||||
- pipeline:
|
||||
name: cpanel-audit-pipeline
|
||||
|
||||
- pipeline:
|
||||
name: cpanel-all-log
|
||||
|
||||
|
||||
|
||||
cpanel-access-pipeline:
|
||||
source:
|
||||
pipeline:
|
||||
name: cpanel-log-router
|
||||
|
||||
processor:
|
||||
- grok:
|
||||
break_on_match: false
|
||||
match:
|
||||
log:
|
||||
- '^%{IP:client_ip} %{NOTSPACE:identity} %{NOTSPACE:user} \[%{GREEDYDATA:timestamp}\] "%{WORD:method} %{URIPATHPARAM:url} HTTP/%{NUMBER:http_version}" %{INT:http_status:int} %{NONNEGINT:bytes:long} "%{NOTSPACE:referrer}" "%{GREEDYDATA:user_agent}" "%{NOTSPACE:auth_method}" "%{GREEDYDATA:x_forwarded_for}" %{POSINT:service_port:int}$'
|
||||
target_key: parsed
|
||||
|
||||
- date:
|
||||
match:
|
||||
- key: /parsed/timestamp
|
||||
patterns:
|
||||
- "MM/dd/yyyy:HH:mm:ss Z"
|
||||
destination: "@timestamp"
|
||||
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts:
|
||||
- "https://brilliant-opensearch-cluster-master:9200"
|
||||
insecure: false
|
||||
username: logger
|
||||
password: "Enp129s0f0-l0gin"
|
||||
index_type: custom
|
||||
cert: /usr/share/dataprepper/cert/root-ca.pem
|
||||
index: cpanel-access-log-%{yyyy.MM.dd}
|
||||
|
||||
cpanel-system-syslog-pipeline:
|
||||
source:
|
||||
pipeline:
|
||||
name: cpanel-log-router
|
||||
processor:
|
||||
- drop_events:
|
||||
drop_when: '/log_source != "system.syslog"'
|
||||
- grok:
|
||||
break_on_match: false
|
||||
match:
|
||||
log:
|
||||
- '^%{TIMESTAMP_ISO8601:timestamp} %{NOTSPACE:hostname} %{DATA:process}\[(?:%{POSINT:pid})\]: %{GREEDYDATA:message}$'
|
||||
target_key: parsed
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts:
|
||||
- "https://brilliant-opensearch-cluster-master:9200"
|
||||
insecure: false
|
||||
username: logger
|
||||
password: "Enp129s0f0-l0gin"
|
||||
index_type: custom
|
||||
cert: /usr/share/dataprepper/cert/root-ca.pem
|
||||
index: cpanel-system-syslog-%{yyyy.MM.dd}
|
||||
|
||||
cpanel-audit-pipeline:
|
||||
source:
|
||||
pipeline:
|
||||
name: cpanel-log-router
|
||||
processor:
|
||||
- drop_events:
|
||||
drop_when: '/log_source != "audit.audit.log"'
|
||||
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts:
|
||||
- "https://brilliant-opensearch-cluster-master:9200"
|
||||
insecure: false
|
||||
username: logger
|
||||
password: "Enp129s0f0-l0gin"
|
||||
index_type: custom
|
||||
cert: /usr/share/dataprepper/cert/root-ca.pem
|
||||
index: cpanel-audit-log-%{yyyy.MM.dd}
|
||||
|
||||
|
||||
cpanel-all-log:
|
||||
source:
|
||||
pipeline:
|
||||
name: cpanel-log-router
|
||||
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts:
|
||||
- "https://brilliant-opensearch-cluster-master:9200"
|
||||
insecure: false
|
||||
username: logger
|
||||
password: "Enp129s0f0-l0gin"
|
||||
index_type: custom
|
||||
cert: /usr/share/dataprepper/cert/root-ca.pem
|
||||
index: cpanel-all-log-%{yyyy.MM.dd}
|
||||
|
||||
|
||||
# -- Data Prepper ports
|
||||
ports:
|
||||
# -- The port that the source is running on. Default value is 2021. Valid options are between 0 and 65535.
|
||||
# https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/http-source/
|
||||
- name: http-source
|
||||
port: 2021
|
||||
# # -- The port that the otel_trace_source source runs on. Default value is 21890.
|
||||
# # https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-trace-source/
|
||||
# - name: otel-traces
|
||||
# port: 21890
|
||||
# # -- The port that the OpenTelemtry metrics source runs on. Default value is 21891.
|
||||
# # https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-metrics-source/
|
||||
# - name: otel-metrics
|
||||
# port: 21891
|
||||
# # -- Represents the port that the otel_logs_source source is running on. Default value is 21892.
|
||||
# # https://opensearch.org/docs/latest/data-prepper/pipelines/configuration/sources/otel-logs-source/
|
||||
# - name: otel-logs
|
||||
# port: 21892
|
||||
|
||||
serviceAccount:
|
||||
# -- Specifies whether a service account should be created
|
||||
create: true
|
||||
# -- Automatically mount a ServiceAccount's API credentials?
|
||||
automount: true
|
||||
# -- Annotations to add to the service account
|
||||
annotations: {}
|
||||
# -- The name of the service account to use.
|
||||
# If not set and create is true, a name is generated using the fullname template
|
||||
name: ""
|
||||
|
||||
podAnnotations: {}
|
||||
podLabels: {}
|
||||
|
||||
podSecurityContext: {}
|
||||
# fsGroup: 2000
|
||||
|
||||
securityContext: {}
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# readOnlyRootFilesystem: true
|
||||
# runAsNonRoot: true
|
||||
# runAsUser: 1000
|
||||
|
||||
initContainers: []
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 2021
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: "nginx"
|
||||
annotations:
|
||||
kubernetes.io/ingress.class: nginx
|
||||
# kubernetes.io/tls-acme: "true"
|
||||
hosts:
|
||||
- paths:
|
||||
- path: /logs
|
||||
pathType: Prefix
|
||||
tls: []
|
||||
# - secretName: chart-example-tls
|
||||
# hosts:
|
||||
# - chart-example.local
|
||||
|
||||
resources:
|
||||
# We usually recommend not to specify default resources and to leave this as a conscious
|
||||
# choice for the user. This also increases chances charts run on environments with little
|
||||
# resources, such as Minikube. If you do want to specify resources, uncomment the following
|
||||
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
|
||||
limits:
|
||||
cpu: 4
|
||||
memory: 6Gi
|
||||
requests:
|
||||
cpu: 2
|
||||
memory: 4Gi
|
||||
|
||||
autoscaling:
|
||||
# done for testing, in production this will be true
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
targetCPUUtilizationPercentage: 75
|
||||
targetMemoryUtilizationPercentage: 75
|
||||
|
||||
# Additional volumes on the output Deployment definition.
|
||||
volumes:
|
||||
- name: root-ca
|
||||
secret:
|
||||
secretName: opensearch-dashboard-ca
|
||||
# - name: foo
|
||||
# secret:
|
||||
# secretName: mysecret
|
||||
# optional: false
|
||||
|
||||
# Additional volumeMounts on the output Deployment definition.
|
||||
volumeMounts:
|
||||
- name: root-ca
|
||||
readOnly: true
|
||||
mountPath: /usr/share/dataprepper/cert
|
||||
# - name: foo
|
||||
# mountPath: "/etc/foo"
|
||||
# readOnly: true
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
Reference in New Issue
Block a user