Convert helm-charts-observability to regular directory
This commit is contained in:
Mosiur Rahman
@@ -0,0 +1,160 @@
|
||||
# Phase 1
|
||||
|
||||
## Infromation for preeper
|
||||
|
||||
- OpenSearch Kubernetes Service name
|
||||
```
|
||||
root@k8s-cp-01:~# k get svc -n observability
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
brilliant-opensearch-cluster-master ClusterIP 10.43.44.32 <none> 9200/TCP,9300/TCP,9600/TCP 4d18h
|
||||
brilliant-opensearch-cluster-master-headless ClusterIP None <none> 9200/TCP,9300/TCP,9600/TCP 4d18h
|
||||
brilliant-opensearch-dashboard-opensearch-dashboards ClusterIP 10.43.225.4 <none> 5601/TCP,9601/TCP 15h
|
||||
root@k8s-cp-01:~# k get svc -n ingress-nginx
|
||||
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
|
||||
ingress-nginx-controller LoadBalancer 10.43.211.10 <pending> 80:31599/TCP,443:31186/TCP 26d
|
||||
ingress-nginx-controller-admission ClusterIP 10.43.204.151 <none> 443/TCP 26d
|
||||
ingress-nginx-controller-metrics ClusterIP 10.43.119.183 <none> 10254/TCP 26d
|
||||
root@k8s-cp-01:~#
|
||||
```
|
||||
- OpenSearch HTTPS port
|
||||
*Mentioned above*
|
||||
- CA certificate
|
||||
*CA Certificate is in the server monitoring/opensearch-pki*
|
||||
*There is another certificate
|
||||
|
||||
```
|
||||
apiVersion: v1
|
||||
data:
|
||||
root-ca.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUY3VENDQTlXZ0F3SUJBZ0lVTC9hTytnYS9YY3lzakFYbGh1WG92d0xLWkJjd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZVXhDekFKQmdOVkJBWVRBa0pFTVE0d0RBWURWUVFJREFWRWFHRnJZVEVPTUF3R0ExVUVCd3dGUkdoaAphMkV4RWpBUUJnTlZCQW9NQ1VKeWFXeHNhV0Z1ZERFYk1Ca0dBMVVFQ3d3U1QzQmxibE5sWVhKamFDQlNiMjkwCklFTkJNU1V3SXdZRFZRUUREQnhDY21sc2JHbGhiblFnVDNCbGJsTmxZWEpqYUNCU2IyOTBJRU5CTUI0WERUSTIKTURjeU16RTFNREV4T1ZvWERUTTJNRGN5TURFMU1ERXhPVm93Z1lVeEN6QUpCZ05WQkFZVEFrSkVNUTR3REFZRApWUVFJREFWRWFHRnJZVEVPTUF3R0ExVUVCd3dGUkdoaGEyRXhFakFRQmdOVkJBb01DVUp5YVd4c2FXRnVkREViCk1Ca0dBMVVFQ3d3U1QzQmxibE5sWVhKamFDQlNiMjkwSUVOQk1TVXdJd1lEVlFRRERCeENjbWxzYkdsaGJuUWcKVDNCbGJsTmxZWEpqYUNCU2IyOTBJRU5CTUlJQ0lqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FnOEFNSUlDQ2dLQwpBZ0VBdXlEYmRaMUFycWd6K1l3SkNUcWlxREtuUzNDRGhHekNxK0h5WDN3clBnV0JvZFBraHJCV3dkNzQ4ZjdqClhZWDNQditQS0F0ZzVGYjdxa2ZZUFZJR0FPZUNiUWxkRlJnZld0S2F2MkdxcHZseUlKcy9OZW5PZzFZQ3IydVUKbDE3UnFPNk1BSGt6d1pNTVFDV1JXMlFiUy9weVY0VHd6dkNxSTR1ZVpzOFJ4MndwbERJdWFPVXdJMEVTMG8zdQpWdDJHMVlCTWlFOUNVTDliQWlYbWQ0UTR4a1NFRFRvOVY1SVk3Z2dHYjFTU0xWU2x6RDJHZElaMGN3VlE2LzcwCnUrQ1FPa005ZXhIRVR2eGpGTmZ4azI2YlRZVUJmNnBNS2VpSDdMbC9ucjZFVVVzTW5DSUR5VGtnK3Zta1M5ZWoKUmY1R0QwQ00rcnl1V1BiRHlHOCtzcDh4d1N1ZkcydDdkUGZRSTUzd1Zzc1BKSkk5b0lsZ0lJWXkyMkFUMlo4RwpKL2dZbmYwMU5LREhSV1dBU2FQaTBLT2dURldqRG5xam94Q0FHZjVGVFRJSHdBQjVsMS8rMXZFbEkyNjQ2M2d4CnBWcTk4Vyt4U3FQQ0xaTW9sWlE5TVkwbjVHeGgvZjhkK0RPb1BVT0lXazJqWXdvS1JGQXJiTlNTNnkySnVrS3YKSWNvMnN4YUhqZkhrdDdqdDBkckRLcjUrelI5bndUWDhVN3hBaWlTYkVTekN6OWlvdU9QSEcxMFFtdzl6VVRNMgoxdzJmMmlQRWpvbFpBVjFBaFU3dzFWNHBzc3JpWDIvNTlDR1dYT1VEcWNnd2MwdWh4djRnWFRndXBCeGcrYU5yCllybWZwNndtejdzN3lmblh3T2xVaVRDRW1Bc2NBemNkMmdCaGwvN1dpcmFRZFVzQ0F3RUFBYU5UTUZFd0hRWUQKVlIwT0JCWUVGSGJhbmo2RkdNVWZ3bEMwSytqY081TTlHd0FwTUI4R0ExVWRJd1FZTUJhQUZIYmFuajZGR01VZgp3bEMwSytqY081TTlHd0FwTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQURnZ0lCCkFBZVVVVUFMTk5FNDM2UEwvZXl6V0c3bmlPdndXODF3UkdVdGdEUnJpY2xFWjh0K3MzUG9MOXRibyszUGN1ekgKakxsUFArYmVLY2tUTEZKVGZZWUN3Z0dWZkdIRXE1ejV1SUt3ZWIyMG9GSEg3ek9DSzF3VVJBSUlPZnFycncyMApjcWNzdTRnK3NENnl0bllWN2Y1T0RFT2RNaXpjVXRrbGpMSjVieXJLSU9OV2FrSTBpd0xLeXFPQWlGeUVSY2xiCjB0ZzhoTDR6dkRLVUZ6dVJycXhxS0padzNWUWUvM1JWUFYwZ0RLUW96VEQ0M3lnQUdGVlVjckx1dHZOK2V5bEEKVDdYMGtNZXJlVVRKd1N6RDVvenhGRXFkaDlLNHZkZTA2a3BuSXpQS2tEanNFZiszeW9WeFVxRGJCeGV3ai9aOApodzhmT3lBWC9jV2UvZlhmWVdPV3RpTlhiQVJRd0pvUWlLb3lacHRuNGpvRXhLWVB2Qzg0RitINEZnZnpidXFkCnNnL2R2SjBxcldGaUlZeWZBRS9NNGM3VTBDUnVsZ1lETXJhbktIUHhxTllVT21pRzM4T0xLTDlFMmlvV002UHYKek82djVvUk5pazFFN3BrRUVMSUFPczR3U2MvdG45MU01TmtpSjgyRXNEQ2ZaVG5FVU9Cd3VsRzZSbW8vYWF0agpCWUJEaHo0dnhmN0ZjNG13WEdXNVZXc1Qzd2ZVckErZ3l3VWo5dlFqTTV6SjEybDl4NU9Dd3pNMTJuQWdSZ1dFCkh4ZDh5TFhkNE9rTWRaRWNVckY5eWtSMGJUZ3AyWnowL3lPR21KbVpieHRNdHlGaTVMcmt6VElodWJNS1l2bncKcWsrbWdaUmNMcXBWclRSYTNiaGdEVzJrSWpFT2I5SHB6NG5MZ3loUVpCMFAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
|
||||
kind: Secret
|
||||
metadata:
|
||||
annotations:
|
||||
kubectl.kubernetes.io/last-applied-configuration: |
|
||||
{"apiVersion":"v1","data":{"root-ca.pem":"LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUY3VENDQTlXZ0F3SUJBZ0lVTC9hTytnYS9YY3lzakFYbGh1WG92d0xLWkJjd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2dZVXhDekFKQmdOVkJBWVRBa0pFTVE0d0RBWURWUVFJREFWRWFHRnJZVEVPTUF3R0ExVUVCd3dGUkdoaAphMkV4RWpBUUJnTlZCQW9NQ1VKeWFXeHNhV0Z1ZERFYk1Ca0dBMVVFQ3d3U1QzQmxibE5sWVhKamFDQlNiMjkwCklFTkJNU1V3SXdZRFZRUUREQnhDY21sc2JHbGhiblFnVDNCbGJsTmxZWEpqYUNCU2IyOTBJRU5CTUI0WERUSTIKTURjeU16RTFNREV4T1ZvWERUTTJNRGN5TURFMU1ERXhPVm93Z1lVeEN6QUpCZ05WQkFZVEFrSkVNUTR3REFZRApWUVFJREFWRWFHRnJZVEVPTUF3R0ExVUVCd3dGUkdoaGEyRXhFakFRQmdOVkJBb01DVUp5YVd4c2FXRnVkREViCk1Ca0dBMVVFQ3d3U1QzQmxibE5sWVhKamFDQlNiMjkwSUVOQk1TVXdJd1lEVlFRRERCeENjbWxzYkdsaGJuUWcKVDNCbGJsTmxZWEpqYUNCU2IyOTBJRU5CTUlJQ0lqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FnOEFNSUlDQ2dLQwpBZ0VBdXlEYmRaMUFycWd6K1l3SkNUcWlxREtuUzNDRGhHekNxK0h5WDN3clBnV0JvZFBraHJCV3dkNzQ4ZjdqClhZWDNQditQS0F0ZzVGYjdxa2ZZUFZJR0FPZUNiUWxkRlJnZld0S2F2MkdxcHZseUlKcy9OZW5PZzFZQ3IydVUKbDE3UnFPNk1BSGt6d1pNTVFDV1JXMlFiUy9weVY0VHd6dkNxSTR1ZVpzOFJ4MndwbERJdWFPVXdJMEVTMG8zdQpWdDJHMVlCTWlFOUNVTDliQWlYbWQ0UTR4a1NFRFRvOVY1SVk3Z2dHYjFTU0xWU2x6RDJHZElaMGN3VlE2LzcwCnUrQ1FPa005ZXhIRVR2eGpGTmZ4azI2YlRZVUJmNnBNS2VpSDdMbC9ucjZFVVVzTW5DSUR5VGtnK3Zta1M5ZWoKUmY1R0QwQ00rcnl1V1BiRHlHOCtzcDh4d1N1ZkcydDdkUGZRSTUzd1Zzc1BKSkk5b0lsZ0lJWXkyMkFUMlo4RwpKL2dZbmYwMU5LREhSV1dBU2FQaTBLT2dURldqRG5xam94Q0FHZjVGVFRJSHdBQjVsMS8rMXZFbEkyNjQ2M2d4CnBWcTk4Vyt4U3FQQ0xaTW9sWlE5TVkwbjVHeGgvZjhkK0RPb1BVT0lXazJqWXdvS1JGQXJiTlNTNnkySnVrS3YKSWNvMnN4YUhqZkhrdDdqdDBkckRLcjUrelI5bndUWDhVN3hBaWlTYkVTekN6OWlvdU9QSEcxMFFtdzl6VVRNMgoxdzJmMmlQRWpvbFpBVjFBaFU3dzFWNHBzc3JpWDIvNTlDR1dYT1VEcWNnd2MwdWh4djRnWFRndXBCeGcrYU5yCllybWZwNndtejdzN3lmblh3T2xVaVRDRW1Bc2NBemNkMmdCaGwvN1dpcmFRZFVzQ0F3RUFBYU5UTUZFd0hRWUQKVlIwT0JCWUVGSGJhbmo2RkdNVWZ3bEMwSytqY081TTlHd0FwTUI4R0ExVWRJd1FZTUJhQUZIYmFuajZGR01VZgp3bEMwSytqY081TTlHd0FwTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQURnZ0lCCkFBZVVVVUFMTk5FNDM2UEwvZXl6V0c3bmlPdndXODF3UkdVdGdEUnJpY2xFWjh0K3MzUG9MOXRibyszUGN1ekgKakxsUFArYmVLY2tUTEZKVGZZWUN3Z0dWZkdIRXE1ejV1SUt3ZWIyMG9GSEg3ek9DSzF3VVJBSUlPZnFycncyMApjcWNzdTRnK3NENnl0bllWN2Y1T0RFT2RNaXpjVXRrbGpMSjVieXJLSU9OV2FrSTBpd0xLeXFPQWlGeUVSY2xiCjB0ZzhoTDR6dkRLVUZ6dVJycXhxS0padzNWUWUvM1JWUFYwZ0RLUW96VEQ0M3lnQUdGVlVjckx1dHZOK2V5bEEKVDdYMGtNZXJlVVRKd1N6RDVvenhGRXFkaDlLNHZkZTA2a3BuSXpQS2tEanNFZiszeW9WeFVxRGJCeGV3ai9aOApodzhmT3lBWC9jV2UvZlhmWVdPV3RpTlhiQVJRd0pvUWlLb3lacHRuNGpvRXhLWVB2Qzg0RitINEZnZnpidXFkCnNnL2R2SjBxcldGaUlZeWZBRS9NNGM3VTBDUnVsZ1lETXJhbktIUHhxTllVT21pRzM4T0xLTDlFMmlvV002UHYKek82djVvUk5pazFFN3BrRUVMSUFPczR3U2MvdG45MU01TmtpSjgyRXNEQ2ZaVG5FVU9Cd3VsRzZSbW8vYWF0agpCWUJEaHo0dnhmN0ZjNG13WEdXNVZXc1Qzd2ZVckErZ3l3VWo5dlFqTTV6SjEybDl4NU9Dd3pNMTJuQWdSZ1dFCkh4ZDh5TFhkNE9rTWRaRWNVckY5eWtSMGJUZ3AyWnowL3lPR21KbVpieHRNdHlGaTVMcmt6VElodWJNS1l2bncKcWsrbWdaUmNMcXBWclRSYTNiaGdEVzJrSWpFT2I5SHB6NG5MZ3loUVpCMFAKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo="},"kind":"Secret","metadata":{"annotations":{},"name":"opensearch-dashboard-ca","namespace":"observability"}}
|
||||
creationTimestamp: "2026-08-03T13:45:31Z"
|
||||
name: opensearch-dashboard-ca
|
||||
namespace: observability
|
||||
resourceVersion: "4485295"
|
||||
uid: 7fba4407-7eb1-456a-a10c-e8500630df2e
|
||||
type: Opaque
|
||||
```
|
||||
- Ingestion username/password
|
||||
*username: logger*
|
||||
*password: Enp129s0f0-l0gin*
|
||||
|
||||
```
|
||||
root@k8s-cp-01:~# k get secret -n observability prepper-creds -o yaml
|
||||
apiVersion: v1
|
||||
data:
|
||||
logger: RW5wMTI5czBmMC1sMGdnM3I=
|
||||
kind: Secret
|
||||
metadata:
|
||||
annotations:
|
||||
kubectl.kubernetes.io/last-applied-configuration: |
|
||||
{"apiVersion":"v1","data":{"logger":"RW5wMTI5czBmMC1sMGdnM3I="},"kind":"Secret","metadata":{"annotations":{},"name":"prepper-creds","namespace":"observability"}}
|
||||
creationTimestamp: "2026-08-04T06:17:58Z"
|
||||
name: prepper-creds
|
||||
namespace: observability
|
||||
resourceVersion: "4604982"
|
||||
uid: 6592aff6-3e6b-45a1-9953-348c647ca8d3
|
||||
type: Opaque
|
||||
root@k8s-cp-01:~#
|
||||
```
|
||||
|
||||
- Target index name
|
||||
*I am not understanding this*
|
||||
|
||||
We will use dedicated ingestion user
|
||||
|
||||
|
||||
# Phase 2
|
||||
|
||||
### Data Prepper pipeline:
|
||||
|
||||
```
|
||||
cpanel-log-pipeline:
|
||||
source:
|
||||
http:
|
||||
path: /logs
|
||||
ssl: true
|
||||
|
||||
|
||||
sink:
|
||||
- opensearch:
|
||||
hosts: ["brilliant-opensearch-cluster-master:9200"]
|
||||
insecure: false
|
||||
username: logger
|
||||
password: ${LOGGER_PASSWORD}
|
||||
index_type: custom
|
||||
index: cpanel-test-%{yyyy.MM.dd}
|
||||
ssl: true
|
||||
```
|
||||
|
||||
log paths of cpanel:
|
||||
|
||||
##### System logs (/var/log)
|
||||
|
||||
|
||||
/var/log/apache2/
|
||||
/var/log/audit/
|
||||
/var/log/auth.log
|
||||
/var/log/bootstrap.log
|
||||
/var/log/chkservd.log
|
||||
/var/log/clamav/
|
||||
/var/log/cpanel-install.log
|
||||
/var/log/cpanel-server-traffic/
|
||||
/var/log/dmesg
|
||||
/var/log/dpkg.log
|
||||
/var/log/exim_mainlog
|
||||
/var/log/exim_paniclog
|
||||
/var/log/exim_rejectlog
|
||||
/var/log/fail2ban.log
|
||||
/var/log/kern.log
|
||||
/var/log/lfd.log
|
||||
/var/log/mail.err
|
||||
/var/log/mail.log
|
||||
/var/log/munin/
|
||||
/var/log/mysql/
|
||||
/var/log/named/
|
||||
/var/log/private/
|
||||
/var/log/quota_enable.log
|
||||
/var/log/sw-cp-server/
|
||||
/var/log/syslog
|
||||
|
||||
##### cPanel logs (/usr/local/cpanel/logs)
|
||||
|
||||
/usr/local/cpanel/logs/access_log
|
||||
/usr/local/cpanel/logs/api_tokens_log
|
||||
/usr/local/cpanel/logs/backup_restore_manager_error_log
|
||||
/usr/local/cpanel/logs/backup_restore_manager_log
|
||||
/usr/local/cpanel/logs/build_locale_databases_log
|
||||
/usr/local/cpanel/logs/cpbackup/
|
||||
/usr/local/cpanel/logs/cpbackup_transport_history.log
|
||||
/usr/local/cpanel/logs/cpbackup_transporter/
|
||||
/usr/local/cpanel/logs/cpbackup_transporter.log
|
||||
/usr/local/cpanel/logs/cpdavd_error_log
|
||||
/usr/local/cpanel/logs/cpdavd_session_log
|
||||
/usr/local/cpanel/logs/cphulkd.log
|
||||
/usr/local/cpanel/logs/cphulkd_errors.log
|
||||
/usr/local/cpanel/logs/cpwrapd_log
|
||||
/usr/local/cpanel/logs/dnsadmin_log
|
||||
/usr/local/cpanel/logs/error_log
|
||||
/usr/local/cpanel/logs/incoming_http_requests.log
|
||||
/usr/local/cpanel/logs/license_log
|
||||
/usr/local/cpanel/logs/login_log
|
||||
/usr/local/cpanel/logs/php-fpm/
|
||||
/usr/local/cpanel/logs/queueprocd.log
|
||||
/usr/local/cpanel/logs/secpol_log
|
||||
/usr/local/cpanel/logs/session_log
|
||||
/usr/local/cpanel/logs/spamd_error_log
|
||||
/usr/local/cpanel/logs/splitlogs_log
|
||||
/usr/local/cpanel/logs/stats_log
|
||||
/usr/local/cpanel/logs/tailwatchd_log
|
||||
/usr/local/cpanel/logs/update_analysis
|
||||
|
||||
|
||||
# Phase 3
|
||||
|
||||
I used values.yaml for creating the resources
|
||||
|
||||
|
||||
Reference in New Issue
Block a user