Author: studyk8s

Traefik ingressroute CRD

https://doc.traefik.io/traefik/routing/providers/kubernetes-crd/

http

apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: foo
  namespace: bar
spec:
  entryPoints:                      # [1]
    - foo
  routes:                           # [2]
  - kind: Rule
    match: Host(`test.example.com`) # [3]
    priority: 10                    # [4]
    middlewares:                    # [5]
    - name: middleware1             # [6]
      namespace: default            # [7]
    services:                       # [8]
    - kind: Service
      name: foo
      namespace: default
      passHostHeader: true
      port: 80
      responseForwarding:
        flushInterval: 1ms
      scheme: https
      serversTransport: transport
      sticky:
        cookie:
          httpOnly: true
          name: cookie
          secure: true
          sameSite: none
      strategy: RoundRobin
      weight: 10
  tls:                              # [9]
    secretName: supersecret         # [10]
    options:                        # [11]
      name: opt                     # [12]
      namespace: default            # [13]
    certResolver: foo               # [14]
    domains:                        # [15]
    - main: example.net             # [16]
      sans:                         # [17]
      - a.example.net
      - b.example.net

tcp

apiVersion: traefik.containo.us/v1alpha1
kind: IngressRouteTCP
metadata:
  name: ingressroutetcpfoo

spec:
  entryPoints:                  # [1]
    - footcp
  routes:                       # [2]
  - match: HostSNI(`*`)         # [3]
    services:                   # [4]
    - name: foo                 # [5]
      port: 8080                # [6]
      weight: 10                # [7]
      terminationDelay: 400     # [8]
      proxyProtocol:            # [9]
        version: 1              # [10]
  tls:                          # [11]
    secretName: supersecret     # [12]
    options:                    # [13]
      name: opt                 # [14]
      namespace: default        # [15]
    certResolver: foo           # [16]
    domains:                    # [17]
    - main: example.net         # [18]
      sans:                     # [19]
      - a.example.net
      - b.example.net
    passthrough: false          # [20]

Traefik 自签名证书

openssl req \
        -newkey rsa:2048 -nodes -keyout tls.key \
        -x509 -days 3650 -out tls.crt

kubectl create secret tls traefik-cert --cert=tls.crt --key=tls.key -n lizhe

apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: whoingressroute2
  namespace: lizhe
spec:
  entryPoints:
    - websecure
  routes:
  - match: Host(`who.lizhe.com`)
    kind: Rule
    services:
    - name: whoami
      port: 80
      sticky:
        cookie:
          httpOnly: false
          name: lizhetoken
  tls:
    secretName: traefik-cert

如果你打算使用 traefik自带的 default cert 可以使用

apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: whoingressroute2
  namespace: lizhe
spec:
  entryPoints:
    - websecure
  routes:
  - match: Host(`who.lizhe.com`)
    kind: Rule
    services:
    - name: whoami
      port: 443
      sticky:
        cookie:
          httpOnly: false
          name: lizhetoken
  tls:
    certResolver: default
    options: {}

CIS Scan

Rancher can run a security scan to check whether Kubernetes is deployed according to security best practices as defined in the CIS Kubernetes Benchmark.

互联网安全中心（CIS）是一个非营利性组织，其制定自己的配置策略基准（即CIS基准），使组织可以改善其安全性和合规性计划及态势。

除了针对操作系统、数据库等，该组织也推出了适用Kubernetes、dockers的Benchmark基准。

Rancher backup

docker启动 minio

docker run -p 9001:9000 --name minio \
    -d --restart=always \
    -e "MINIO_ROOT_USER=admin" \
    -e "MINIO_ROOT_PASSWORD=12345678" \
    -v ~lizhe/minio/data:/data \
    -v ~lizhe/minio/config:/root/.minio \
    minio/minio server /data

Redis cluster on kubernetes

https://charts.bitnami.com/bitnami

仓库中的 redis-cluster 包直接提供了 3个master 和 3个slave的标准集群模式

使用RDM客户端此时是可以正常访问集群模式的，因为 RDM客户端支持

Change host on cluster redirects

不过在这种状态下，绑定 redis 实例的 nodeport 无法使用 python的 redis 包直接访问 需要使用 redis-py-cluster

redis集群在kubernetes 中最主要的麻烦来自于它的 16384 个哈希槽，redis cluster 的客户端需要在各个节点之间进行forward

from rediscluster import RedisCluster
startup_nodes = [{"host": "192.168.194.194", "port": "30002"}]
rc = RedisCluster(startup_nodes=startup_nodes, decode_responses=True, password="f1w6shwtLa")
rc.set("foo", "bar")
print(rc.get("foo"))
print(rc.get("foo"))
print(rc.get("foo"))

还有一个选择是 redis-cluster-proxy

git clone https://github.com/RedisLabs/redis-cluster-proxy.git

make PREFIX=/usr/local/redis_cluster_proxy install 

如果出现错误

../deps/hiredis/libhiredis.a: No such file or directory

root@ubuntu:~/proxy/redis-cluster-proxy# cd deps
root@ubuntu:~/proxy/redis-cluster-proxy/deps# make lua hiredis linenoise

root@ubuntu:~/proxy/docker# cp /usr/local/redis_cluster_proxy/bin/redis-cluster-proxy ./
root@ubuntu:~/proxy/docker# ls
Dockerfile  redis-cluster-proxy
root@ubuntu:~/proxy/docker# cat Dockerfile 
FROM ubuntu
WORKDIR /data
ADD redis-cluster-proxy /usr/local/bin/
EXPOSE 7777
root@ubuntu:~/proxy/docker# 

docker build . -t libaibai/redis-cluster-proxy:v1.0.0

root@ubuntu:~/proxy/k8s# cat configmap.yaml 
---
# Redis-Proxy Config
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-proxy
  namespace: redis-cluster
data:
  proxy.conf: |
    cluster redis-cluster.redis-cluster.svc:6379     # 配置为Redis Cluster Service
    bind 0.0.0.0
    port 7777   # redis-cluster-proxy 对外暴露端口
    threads 8   # 线程数量
    daemonize no  
    enable-cross-slot yes    
    auth f1w6shwtLa   # 配置Redis Cluster 认证密码  
    log-level error

root@ubuntu:~/proxy/k8s# 

root@ubuntu:~/proxy/k8s# cat deploy.yaml 
---
# Redis-Proxy NodePort
apiVersion: v1
kind: Service
metadata:
  name: redis-proxy
  namespace: redis-cluster
spec:
  type: NodePort # 对K8S外部提供服务
  ports:
  - name: redis-proxy
    nodePort: 30001   # 对外提供的端口
    port: 7777
    protocol: TCP
    targetPort: 7777
  selector:
    app: redis-proxy
---
# Redis-Proxy Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-proxy
  namespace: redis-cluster
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis-proxy
  template:
    metadata:
      labels:
        app: redis-proxy
    spec:
      imagePullSecrets:
        - name: harbor
      containers:
        - name: redis-proxy
          image: libaibai/redis-cluster-proxy:v1.0.0
          imagePullPolicy: Always
          command: ["redis-cluster-proxy"]
          args:
            - -c
            - /data/proxy.conf   # 指定启动配置文件
          ports:
            - name: redis-7777
              containerPort: 7777
              protocol: TCP
          volumeMounts:
            - name: redis-proxy-conf
              mountPath: /data/
      volumes:   # 挂载proxy配置文件
        - name: redis-proxy-conf
          configMap:
            name: redis-proxy

root@ubuntu:~/proxy/k8s# 

EFK 最佳实践简单版

Elasticsearch、filebeat 和 kibana 是最常见的用来管理 kubernetes 集群日志的方式。

1. 最简单的是使用 filebeat -> elsticsearch -> kibana

请参考 EFK安装

这种情况下最致命的缺点来自于 Elasticsearch 的吞吐量，提升Elasticsearch吞吐量的办法有很多，磁盘性能，节点的shard数量，master节点的性能，把Elasticsearch单机替换成集群模式，修改index.refresh_interval降低搜索的实时性换取更高的写入性能等等

2. 通过MQ来提升吞吐量 filebeat -> kafka -> logstash -> elasticsearch – > kibana

请参考 EFK Logstash Kafka

3. 为日志系统增加安全性验证

在 EFK安装 中提到了如何对单机版本的 Elasticsearch 和 kibana 使用 xpack 添加权限

问题在于 如果在 Elasticsearch 集群中 添加 xpack 权限的话，需要对 Elasticsearch 集群添加 https 证书验证
在kubernetes中进行此步骤会比较复杂，当然此种方法安全性是最高的，节点之间的通讯也通过了https加密，不过鉴于kubernetes集群内网中的Elasticsearch往往都是通过内网来通讯的，所以这点在很多情况下可以被忽略。

4 更简单的方式

本文主要为了提供一种既可以满足 大吞吐量 又 提供适度的安全性 的方案

filebeat -> kafka -> logstash -> elasticsearch cluster -> kibana -> traefik basic auth

以下是具体步骤

1. Elasticsearch集群的安装

可以直接通过helm https://helm.elastic.co/

下面的values会创建5个master节点，当然如果需要 data、client 等节点可以通过values设定

---
  antiAffinity: "soft"
  replicas: "5"
  resources: 
    requests: 
      cpu: "200m"
  volumeClaimTemplate: 
    resources: 
      requests: 
        storage: "3Gi"

2. 安装kafka

3. 安装 filebeat 将抓取到的日志推送到 kafka

/usr/share/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  paths:
    - /run/containerd/io.containerd.runtime.v1.linux/**/*.log
    - /run/containerd/io.containerd.runtime.v1.linux/k8s.io/*/rootfs/usr/local/tomcat/app.log

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

processors:
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

output.kafka:
  hosts: ["kafkasvc.efk.svc:9093"]
  topic: 'estopic'
  partition.round_robin:
    reachable_only: false
  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

4. 安装 Logstash ， Logstash 从 kafka 读取数据，然后推送给 Elasticsearch

/usr/share/logstash/pipeline/logstash.conf

input {
  kafka {
    bootstrap_servers => ["kafkasvc.efk.svc:9092"]
    group_id => "es-group"
    topics => ["estopic"] 
    codec => json
 }
}
filter {

}
output {
  elasticsearch {
    hosts => "http://elasticsearch-master.efk.svc:9200"
    index => "kafka‐%{+YYYY.MM.dd}"
 }
}

/usr/share/logstash/config/logstash.yml

http.host: "0.0.0.0"

5. 安装 kibana

kibana 环境变量

SERVER_HOST 	0.0.0.0
NODE_OPTIONS 	--max-old-space-size=1800
ELASTICSEARCH_HOSTS 	http://elasticsearch-master:9200

6. 安装并且配置 traefik

https://helm.traefik.io/traefik

创建中间件 关于密码的生成请参考 Traefik basic auth

# Declaring the user list
apiVersion: traefik.containo.us/v1alpha1
kind: Middleware
metadata:
  name: dev-auth
  namespace: efk
spec:
  basicAuth:
    secret: authsecret

---
# Note: in a kubernetes secret the string (e.g. generated by htpasswd) must be base64-encoded first.
# To create an encoded user:password pair, the following command can be used:
# htpasswd -nb user password | openssl base64

apiVersion: v1
kind: Secret
metadata:
  name: authsecret
  namespace: efk

data:
  users: |2
    dXNlcjE6JGFwcjEkeUw0OEFEWm8kazFmWW1RVTUuZjhwby9SSld6UFdMLwp1c2Vy
    MjokYXByMSRJWXBvc1lSMCRuZWJGZ201ejEzdEh0VFVGUktZeVAxCnVzZXIzOiRh
    cHIxJFZLTDVuckIzJG9iVUduR0FXSTYxMFNPUHZZblM2Vy4K

创建ingressroute

kind: IngressRoute
metadata:
  name: kibanaingress
  namespace: efk
spec:
  entryPoints:
    - web
  routes:
    - match: Host(`kibana.lizhe.com`)
      kind: Rule
      services:
        - name: kibanasvc
          port: 5601
      middlewares:
        - name: dev-auth

通过 ingress 访问 就会得到下面的 basic auth 验证

如何为Elasticsearch添加 data节点

使用同样的 helm

添加以下values

roles:
  master: "false"
  ingest: "false"
  data: "true"
  remote_cluster_client: "true"
  ml: "true"

replicas: 2

resources:
  requests:
    cpu: "200m"
    memory: "1Gi"
  limits:
    cpu: "1000m"
    memory: "2Gi"

antiAffinity: "soft"
nodeGroup: "data"

volumeClaimTemplate:
  accessModes: [ "ReadWriteOnce" ]
  resources:
    requests:
      storage: 3Gi

可以看到 Elasticsearch 节点 从5个master 变成了7个，多出了2个data节点

Elasticsearch cluster xpack 权限验证

单机版本的xpack权限验证很容易打开

/usr/share/elasticsearch/config/elasticsearch.yml

xpack.security.enabled: true
xpack.license.self_generated.type: basic
xpack.security.transport.ssl.enabled: true
network.host: 0.0.0.0

bin/elasticsearch-setup-passwords interactive

下面是一个 cluster 例子

这里我们使用的namespace是 efk

1. es_master_configmap.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: efk
  name: elasticsearch-master-config
  labels:
    app: elasticsearch
    role: master
data:
  elasticsearch.yml: |-
    cluster.name: ${CLUSTER_NAME}
    node.name: ${NODE_NAME}
    discovery.seed_hosts: ${NODE_LIST}
    cluster.initial_master_nodes: ${MASTER_NODES}

    network.host: 0.0.0.0

    node:
      master: true
      data: false
      ingest: false

    xpack.security.enabled: true
    xpack.monitoring.collection.enabled: true
---

2. es_master_service.yaml

---
apiVersion: v1
kind: Service
metadata:
  namespace: efk
  name: elasticsearch-master
  labels:
    app: elasticsearch
    role: master
spec:
  ports:
  - port: 9200
    name: http
  - port: 9300
    name: transport
  selector:
    app: elasticsearch
    role: master
---

3. es_master_data_pvc.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    volume.beta.kubernetes.io/storage-provisioner: driver.longhorn.io
  finalizers:
  - kubernetes.io/pvc-protection
  labels:
    cattle.io/creator: norman
  name: elasticsearchdata
  namespace: efk
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 3Gi
  storageClassName: longhorn
  volumeMode: Filesystem

4. es_master_deployment.yaml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  namespace: efk
  name: elasticsearch-master
  labels:
    app: elasticsearch
    role: master
spec:
  replicas: 1
  selector:
    matchLabels:
      app: elasticsearch
      role: master
  template:
    metadata:
      labels:
        app: elasticsearch
        role: master
    spec:
      containers:
      - name: elasticsearch-master
        image: docker.elastic.co/elasticsearch/elasticsearch:7.13.2
        env:
        - name: CLUSTER_NAME
          value: elasticsearch
        - name: NODE_NAME
          value: elasticsearch-master
        - name: NODE_LIST
          value: elasticsearch-master,elasticsearch-data,elasticsearch-client
        - name: MASTER_NODES
          value: elasticsearch-master
        - name: "ES_JAVA_OPTS"
          value: "-Xms256m -Xmx256m"
        ports:
        - containerPort: 9300
          name: transport
        volumeMounts:
        - name: config
          mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
          readOnly: true
          subPath: elasticsearch.yml
        - name: elasticsearchdata
          mountPath: /usr/share/elasticsearch/data
      volumes:
      - name: config
        configMap:
          name: elasticsearch-master-config
      - name: elasticsearchdata
        persistentVolumeClaim:
          claimName: elasticsearchdata
      initContainers:
      - name: increase-vm-max-map
        image: busybox
        command: ["sysctl", "-w", "vm.max_map_count=262144"]
        securityContext:
          privileged: true
---

Elasticsearch helm安装cluster

这里使用的是 https://helm.elastic.co/ 仓库中的官方包

因为我是单机节点而且资源有限，所以添加了 values

antiAffinity = soft
volumeClaimTemplate.resources.requests.storage = 3Gi
resources.requests.cpu = 500m
replicas = 5

可以看到5个节点都是健康的

curl localhost:9200/_nodes/process?pretty

Vagrant One-time ssh password

# To view in Web UI
path "sys/mounts" {
  capabilities = [ "read", "update" ]
}

# To configure the SSH secrets engine
path "ssh/*" {
  capabilities = [ "create", "read", "update", "delete", "list" ]
}

# To enable secrets engines
path "sys/mounts/*" {
  capabilities = [ "create", "read", "update", "delete" ]
}

/ $ vault secrets enable ssh
Success! Enabled the ssh secrets engine at: ssh/
/ $ 
/ $ vault write ssh/roles/otp_key_role \
>     key_type=otp \
>     default_user=ubuntu \
>     cidr_list=0.0.0.0/0
Success! Data written to: ssh/roles/otp_key_role
/ $ 
/ $ 

/ $ vault secrets enable ssh
Success! Enabled the ssh secrets engine at: ssh/
/ $ 
/ $ vault write ssh/roles/otp_key_role \
>     key_type=otp \
>     default_user=ubuntu \
>     cidr_list=0.0.0.0/0
Success! Data written to: ssh/roles/otp_key_role
/ $ 
/ $ vault auth enable userpass
Success! Enabled userpass auth method at: userpass/
/ $ 
/ $ vault write auth/userpass/users/bob password="training" policies="ssh-otp"
Success! Data written to: auth/userpass/users/bob
/ $ 

Vault dynamic secrets MongoDB

dynamic secret是在访问时生成的，也就是说 dynamic secret 在被读取之前是不存在的，因此没有人窃取它们或其他客户使用相同机密的风险。同时因为Vault具有内置的吊销机制，所以dynamic secret可以在使用后立即吊销，从而将机密存在的时间减至最少。

官方入门文档中提供了一个 aws engine 的说明

https://learn.hashicorp.com/tutorials/vault/getting-started-dynamic-secrets?in=vault/getting-started

同时提供了

postgres https://learn.hashicorp.com/tutorials/vault/database-secrets?in=vault/secrets-management

mongodb https://learn.hashicorp.com/tutorials/vault/database-mongodb?in=vault/secrets-management

下面是一个 MongoDB的例子

启动一个数据库

docker run -d \
    -p 0.0.0.0:27017:27017 -p 0.0.0.0:28017:28017 \
    --name=mongodb \
    -e MONGO_INITDB_ROOT_USERNAME="mdbadmin" \
    -e MONGO_INITDB_ROOT_PASSWORD="hQ97T9JJKZoqnFn2NXE" \
    mongo

Configure MongoDB secrets engine

vault write mongodb/config/mongo-test \
      plugin_name=mongodb-database-plugin \
      allowed_roles="tester" \
      connection_url="mongodb://mdbadmin:hQ97T9JJKZoqnFn2NXE@192.168.194.193:27017/admin?tls=false" \
      username="mdbadmin" \
      password="hQ97T9JJKZoqnFn2NXE"

vault write mongodb/roles/tester \
    db_name=mongo-test \
    creation_statements='{ "db": "admin", "roles": [{ "role": "readWrite" }, {"role": "read", "db": "foo"}] }' \
    default_ttl="1h" \
    max_ttl="24h"

vault read mongodb/creds/tester

Key                Value
---                -----
lease_id           mongodb/creds/tester/7GPR6sRrPloRvAn5yGzoYPjI
lease_duration     1h
lease_renewable    true
password           rKzEPbGPw-6mG19-4hKL
username           v-root-tester-BNUUEQrC8iyHGRDeK7Bu-1624377059

登录到mongodb查看新用户

docker exec -it mongodb mongo \
    --username v-root-tester-BNUUEQrC8iyHGRDeK7Bu-1624377059 \
    --password rKzEPbGPw-6mG19-4hKL

show dbs

UI 上的内容