apt update && apt upgrade -y && apt install -y curl ca-certificates
timedatectl set-timezone Asia/Shanghai

adduser evan
usermod -aG sudo evan

curl -fsSL https://get.docker.com | sh
systemctl enable --now docker
usermod -aG docker evan

sudo -iu evan
mkdir -p ~/vaultwarden/{vw-data,caddy-data,caddy-config}
cd ~/vaultwarden

services:
  vaultwarden:
    image: vaultwarden/server:latest
    container_name: vaultwarden
    restart: unless-stopped
    environment:
      DOMAIN: "https://${VAULT_HOST}"
      ADMIN_TOKEN: ${ADMIN_TOKEN}
      SIGNUPS_ALLOWED: "false"
      SMTP_HOST: "smtp.resend.com"
      SMTP_FROM: ${SMTP_FROM}
      SMTP_PORT: "465"
      SMTP_SECURITY: "force_tls"
      SMTP_USERNAME: "resend"
      SMTP_PASSWORD: ${SMTP_PASSWORD}
    deploy:
      resources:
        limits:
          cpus: '0.5'
          memory: 512M
    volumes:
      - ./vw-data:/data
    networks: [internal]

  caddy:
    image: caddy:2
    container_name: caddy
    restart: unless-stopped
    environment:
      VAULT_HOST: ${VAULT_HOST}
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./Caddyfile:/etc/caddy/Caddyfile:ro
      - ./caddy-data:/data
      - ./caddy-config:/config
    networks: [internal]

networks:
  internal:
    driver: bridge

vim .env

VAULT_HOST="vault.example.com"
SMTP_FROM="vault@notify.example.com"
SMTP_PASSWORD="re_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
ADMIN_TOKEN='$argon2id$v=19$m=65540,t=3,p=4$<salt>$<hash>'

docker run --rm -it vaultwarden/server /vaultwarden hash

{$VAULT_HOST} {
    encode zstd gzip
    reverse_proxy vaultwarden:80
}

docker compose up -d
docker compose logs -f

sudo apt install -y restic

cat > /root/.restic-env <<'EOF'
B2_ACCOUNT_ID=<key-id>
B2_ACCOUNT_KEY=<application-key>
RESTIC_REPOSITORY=b2:my-vaultwarden-backup:/
RESTIC_PASSWORD_FILE=/root/.restic-password
EOF
chmod 600 /root/.restic-env

echo '<a-strong-passphrase>' > /root/.restic-password
chmod 600 /root/.restic-password

set -a; source /root/.restic-env; set +a
restic init

#!/usr/bin/env bash
set -euo pipefail
set -a; source /root/.restic-env; set +a

# 任何退出路径都保证容器恢复运行
trap 'docker start vaultwarden >/dev/null 2>&1 || true' EXIT

# 短暂停机（~5s）拿到一致性快照
docker stop vaultwarden >/dev/null

# 备份整个部署目录：vw-data + Caddyfile + docker-compose.yml + .env + caddy-data
# 用相对路径，snapshot 里只记 vaultwarden/，恢复时落点干净
cd /home/evan && restic backup vaultwarden \
  --tag vaultwarden \
  --exclude='vaultwarden/vw-data/icon_cache'

# 滚动保留：日 7 / 周 4 / 月 12
restic forget --prune \
  --keep-daily 7 --keep-weekly 4 --keep-monthly 12

chmod +x /root/vaultwarden-backup.sh

sudo crontab -e

# 拉一份最新快照到临时目录
restic snapshots
restic restore latest --target /tmp/vw-restore

# /tmp/vw-restore/vaultwarden/ 是完整部署，cd 进去 docker compose up -d 就能起
# 用 master password 登录 → 打开任意条目 → 字段能看到明文 = 成功

机场订阅 · 自建节点
      ↓
Sub-Store · 合并 / 去重 / 改名 / 过滤 / 分组
      ↓  纯节点订阅
subconverter (+ sub-web GUI) · 套 ACL4SSR · 转目标格式
      ↓
客户端 · Mihomo / Surge / sing-box

apt update && apt upgrade -y && apt install -y curl ca-certificates
timedatectl set-timezone Asia/Shanghai
adduser evan && usermod -aG sudo evan
curl -fsSL https://get.docker.com | sh
systemctl enable --now docker
usermod -aG docker evan

sudo -iu evan
mkdir -p ~/sherpa/{sub-store-data,subconverter/{snippets,rules}}
cd ~/sherpa

x-logging: &log
  driver: json-file
  options: { max-size: "10m", max-file: "3" }

services:
  sub-store:
    image: xream/sub-store:latest
    container_name: sub-store
    restart: unless-stopped
    ports: ["127.0.0.1:3001:3001"]
    volumes: [./sub-store-data:/opt/app/data]
    environment:
      - "SUB_STORE_FRONTEND_BACKEND_PATH=${SUB_STORE_FRONTEND_BACKEND_PATH:?missing in .env}"
      - SUB_STORE_FRONTEND_HOST=0.0.0.0
      - SUB_STORE_FRONTEND_PORT=3001
      - SUB_STORE_DATA_BASE_PATH=/opt/app/data
    networks: [sherpa-net]
    logging: *log

  # MetaCubeX fork 跟进 Hysteria2 / TUIC / Reality;tindy2013 原版更新慢
  subconverter:
    image: ghcr.io/metacubex/subconverter:latest
    container_name: subconverter
    restart: unless-stopped
    ports: ["127.0.0.1:25500:25500"]
    volumes:
      - ./subconverter/pref.toml:/base/pref.toml:ro
      - ./subconverter/snippets:/base/snippets:ro
      - ./subconverter/rules:/base/rules:ro
    networks: [sherpa-net]
    logging: *log

  sub-web:
    image: careywong/subweb:latest
    container_name: sub-web
    restart: unless-stopped
    environment: [API_URL=https://${SUB_DOMAIN}]
    ports: ["127.0.0.1:25580:80"]
    depends_on: [subconverter]
    networks: [sherpa-net]
    logging: *log

# 显式 name,Caddy 子项目 external 引用;容器之间用服务名通信，不走宿主机 loopback
networks:
  sherpa-net:
    driver: bridge
    name: sherpa-net

cat > .env <<EOF
SUB_DOMAIN=sub.example.com
SUB_STORE_FRONTEND_BACKEND_PATH=/api-$(openssl rand -hex 16)
SSH_HOST=evan@<vps-ip>
EOF
chmod 600 .env

cat > subconverter/pref.toml <<EOF
[common]
api_mode = true
api_access_token = "$(openssl rand -hex 32)"

[node_pref]
udp_flag = true
filter_deprecated_nodes = true
append_sub_userinfo = true
clash_use_new_field_name = true

[managed_config]
write_managed_config = true
# 必须真实公网域名,客户端按这个 URL 自动刷新;写 localhost 客户端更新不了
managed_config_prefix = "https://sub.example.com"

[emojis]
add_emoji = true
remove_old_emoji = true

[ruleset]
enabled = true
overwrite_original_rules = true
EOF
chmod 600 subconverter/pref.toml

docker compose up -d
curl -s http://127.0.0.1:25500/version       # subconverter vx.x.x backend
curl -sI http://127.0.0.1:3001/ | head -1    # HTTP/1.1 200 OK

sub.example.com {
    encode gzip
    @blocked not header User-Agent *Clash* *Quantumult* *Surge* *Shadowrocket* *Stash* *Loon* *Mihomo* *sing-box* *curl* *wget*
    respond @blocked 403
    reverse_proxy 127.0.0.1:25500
}

# caddy/docker-compose.yml
services:
  caddy:
    image: caddy:2-alpine
    restart: unless-stopped
    ports: ["80:80", "443:443"]
    volumes:
      - ./Caddyfile:/etc/caddy/Caddyfile:ro
      - caddy_data:/data
      - caddy_config:/config
    networks: [caddy-net, sherpa-net]

networks:
  caddy-net: { driver: bridge, name: caddy-net }
  sherpa-net: { external: true, name: sherpa-net }   # 引用主项目同名网络

volumes: { caddy_data: {}, caddy_config: {} }

curl -i https://sub.example.com/version            # 403,UA 被挡
curl -iA "Mihomo" https://sub.example.com/version  # 200

SOCK=/tmp/sherpa-tunnel-$(id -u).sock
ssh -S "$SOCK" -O check sherpa 2>/dev/null || \
  ssh -fN -M -S "$SOCK" sherpa     # 启动(已有则跳过)
ssh -S "$SOCK" -O check sherpa     # 查
ssh -S "$SOCK" -O exit  sherpa     # 关

https://sub.example.com/sub
  ?target=clash
  &url=<URL-encoded 第 2 层 URL>
  &config=<URL-encoded 远程 config>
  &access_token=<pref.toml 的 api_access_token>
  &emoji=true&list=false&udp=true

git clone --depth=1 https://github.com/ACL4SSR/ACL4SSR.git ~/sherpa/subconverter/rules/ACL4SSR
# config 改成 config/ACL4SSR/Clash/config/ACL4SSR_Online_Full.ini

# vi default.conf
server {
    # for better compatibility it's recommended
    # to use the same port for http/3 and https
    # 浏览器首次只能走 TCP，因此必须提供一个 HTTPS 入口，再通过 `Alt-Svc` 升级到 HTTP/3
    listen 443 ssl;
    # Quick UDP Internet Connections
    # quic 关键字表示使用 UDP 监听，承载 QUIC/HTTP-3
    # reuseport 关键字表示复用端口，允许多个 NGINX 实例共享同一端口
    listen 443 quic reuseport;

    http2 on;
    # 启用 HTTP/3，默认是 on
    http3 on;

    server_name localhost;
    ssl_certificate /etc/ssl/certs/localhost.pem;
    ssl_certificate_key /etc/ssl/certs/localhost-key.pem;

    # 启用 TLS 允许 0-RTT（Early Data），允许客户端在 TLS 1.3 的重连阶段携带早期数据
    # 对 HTTP/3 非必需，但 QUIC 天生支持 0-RTT；此处保持一致
    ssl_early_data on;

    # 仅启用 TLSv1.3
    ssl_protocols TLSv1.3;
    ssl_ciphers HIGH:!aNULL:!MD5;
    ssl_prefer_server_ciphers on;

    # 添加 Alt-Svc 头，向客户端声明可升级到 HTTP/3
    # Alt-Svc 头告诉浏览器"该资源在同一主机的 UDP/443 上能用 h3 协议"，浏览器会自动重试
    # ma max-age 缓存时间，86400 秒 = 1 天
    # always 关键字保证即使发生 4xx/5xx 也发送 Alt-Svc，便于调试
    add_header alt-svc 'h3=":443"; ma=86400' always;

    location / {
        root /usr/share/nginx/html;
        index  index.html index.htm;
    }
}

# docker rm -f nginx-ssl

# 添加监听 UDP 端口
docker run -d --name nginx-ssl \
  -p 443:443 \
  -p 443:443/udp \
  -v ./default.conf:/etc/nginx/conf.d/default.conf:ro \
  -v ./localhost.pem:/etc/ssl/certs/localhost.pem:ro \
  -v ./localhost-key.pem:/etc/ssl/certs/localhost-key.pem:ro \
  -v ./html/:/usr/share/nginx/html/:ro \
  nginx:alpine

# docker container restart nginx-ssl

# 使用支持 HTTP/3 的 curl
# docker run -ti --rm alpine/curl-http3 curl -V
docker run -ti --network host --rm alpine/curl-http3 curl -I --http3 https://localhost/

HTTP/3 200
server: nginx/1.29.0
date: Fri, 27 Jun 2025 03:26:09 GMT
content-type: text/html
content-length: 656
last-modified: Fri, 27 Jun 2025 03:17:07 GMT
etag: "685e0d33-290"
alt-svc: h3=":443"; ma=86400
accept-ranges: bytes

brew install mkcert
mkcert -install
mkcert localhost
# The certificate is at "./localhost.pem" and the key at "./localhost-key.pem" ✅

# vi default.conf
server {
    listen 443 ssl;

    server_name localhost;
    ssl_certificate /etc/ssl/certs/localhost.pem;
    ssl_certificate_key /etc/ssl/certs/localhost-key.pem;

    # 只允许 TLSv1.2 和 TLSv1.3 协议，显式禁止 TLSv1.0 和 TLSv1.1 协议
    ssl_protocols TLSv1.2 TLSv1.3;
    # 只保留高强度、带身份验证、且不使用 MD5 的 TLS 1.2（及以下）套件
    ssl_ciphers HIGH:!aNULL:!MD5;
    # 优先使用服务器端加密套件
    ssl_prefer_server_ciphers on;

    location / {
        root /usr/share/nginx/html;
        index  index.html index.htm;
    }
}

docker run -d --name nginx-ssl \
  -p 8443:443 \
  -v ./default.conf:/etc/nginx/conf.d/default.conf:ro \
  -v ./localhost.pem:/etc/ssl/certs/localhost.pem:ro \
  -v ./localhost-key.pem:/etc/ssl/certs/localhost-key.pem:ro \
  nginx:alpine

# docker rm -f nginx-ssl
# docker container restart nginx-ssl

curl -I https://localhost:8443/

HTTP/1.1 200 OK
Server: nginx/1.29.0
Date: Fri, 27 Jun 2025 01:54:36 GMT
Content-Type: text/html
Content-Length: 615
Last-Modified: Tue, 24 Jun 2025 17:57:38 GMT
Connection: keep-alive
ETag: "685ae712-267"
Accept-Ranges: bytes

# -v 显示详细信息，查看 TLS 版本
curl -I https://localhost:8443/ -v

...
* SSL connection using TLSv1.3 / AEAD-AES256-GCM-SHA384 / [blank] / UNDEF
...

客户端                           服务器
   |<--- SYN → SYN/ACK → ACK --->| 建立可靠的 TCP 连接

   |--- ClientHello (TLS)    --->| 浏览器发起 TLS 握手，告诉服务器自己支持的协议版本、密码套件、随机数（Client Random）等信息
   |<-- ServerHello + 证书     ---|
   |--- 密钥协商/Finished ------->|
   |<-- Finished ----------------|

   TLS 信道建立完成
   |--- HTTP GET / ------------->|
   |<-- HTTP/1.1 200 OK ---------|

openssl version
# 看看证书
openssl x509 -in localhost.pem -noout -text

Certificate:
｜- Data:
  ｜- Version: 3
  ｜- Serial Number: ...
  ｜- Signature Algorithm: sha256WithRSAEncryption
  ｜- Issuer: O=mkcert development CA, OU=..., CN=mkcert ...
  ｜- Validity:
    ｜- Not Before: Jun 27 03:39:39 2025 GMT
    ｜- Not After : Sep 27 03:39:39 2027 GMT
  ｜- Subject: O=mkcert development CA, OU=...
  ｜- Subject Public Key Info: <网站的公钥>
    ｜- Public Key Algorithm: rsaEncryption
      ｜- Public-Key: (2048 bit)
      ｜- Modulus: <模数>
      ｜- Exponent: 65537 (0x10001)
   |- X509v3 extensions
｜- Signature Algorithm: sha256WithRSAEncryption
｜- Signature Value: <颁发机构（CA）用自己私钥算出的数字签名，浏览器用 CA 公钥验证>

# 导出公钥
openssl x509 -in localhost.pem -pubkey -noout > pubkey.pem
# 查看导出的公钥
openssl pkey -pubin -in pubkey.pem -text -noout

# 看看私钥里有什么
openssl rsa -in localhost-key.pem -text -noout

Private-Key: (2048 bit, 2 primes)
# 模 n = p × q
modulus: ...
# e 公钥指数
publicExponent: 65537 (0x10001)

# m = c^d mod n
privateExponent: ...

# p 和 q 是两个大质数
prime1: ...
prime2: ...

# exponent1 = d mod (p-1)
exponent1: ...
# exponent2 = d mod (q-1)
exponent2: ...
# coefficient = inv(q) mod p
coefficient: ...

tar -xvf dump.tar.gz
mongorestore --uri="mongodb://root:root@10.130.0.12/?&authMechanism=SCRAM-SHA-1"

// 插入
db.fruit.insertOne({ name: "apple" })
db.fruit.insertMany([{ name: "apple" }, { name: "pear" }, { name: "orange" }])

// 查询
db.customers.find({ username: "fmiller", name: "Elizabeth Ray" })
db.customers.find({ username: /^f/ })
db.customers.find({ $or: [{ username: /^f/ }, { name: /^E/ }] })

// 投影（projection）
db.customers.find({ username: /^f/ }, { name: 0, email: 0 })   // 排除
db.customers.find({ username: /^f/ }, { _id: 1, name: 1 })     // 仅返回

// 删除
db.customers.remove({ username: "abrown" })

// 更新
db.customers.updateOne({ username: "fmiller" }, { $set: { from: "China" } })
db.customers.updateMany({ username: "fmiller" }, { $set: { from: "China" } })

// 表 / 库管理
db.fruit.drop()
show collections
db.dropDatabase()
show dbs

pipeline = [stage1, stage2, ...]
db.<collection>.aggregate(pipeline, { option })

// 全集合总计
db.orders.aggregate([
    { $group: { _id: null, total: { $sum: "$total" } } }
])
// { _id: null, total: NumberDecimal("44019609") }

// 2019 Q1 已完成订单总金额（订单金额 + 运费）和订单数
db.orders.aggregate([
    {
        $match: {
            status: "completed",
            orderDate: { $gte: ISODate("2019-01-01"), $lt: ISODate("2019-04-01") }
        }
    },
    {
        $group: {
            _id: null,
            total: { $sum: "$total" },
            shippingFee: { $sum: "$shippingFee" },
            count: { $sum: 1 }
        }
    },
    {
        $project: {
            grandTotal: { $add: ["$total", "$shippingFee"] },
            count: 1,
            _id: 0
        }
    }
])
// { count: 5875, grandTotal: NumberDecimal("2636376") }

mkdir -p runtime/data_db{1,2,3} \
      && mkdir -p runtime/data_configdb{1,2,3}

hostname -f

rs.status()

db.contacts.aggregate([
  { $lookup: {
      from: "groups",
      localField: "groups_ids",
      foreignField: "groups_id",
      as: "groups"
  }}
])

{ w: <value>, j: <boolean>, wtimeout: <number> }

db.test.insert({ count: 2 }, { writeConcern: { w: "majority" } })
db.test.insert({ count: 2 }, { writeConcern: { w: 3 } })
db.test.insert({ count: 2 }, { writeConcern: { w: 4 } })
// [Error] 100 - Not enough data-bearing nodes

// 配置延迟节点（5s 延迟、不参选）
conf = rs.conf()
conf.members[2].slaveDelay = 5
conf.members[2].priority = 0
rs.reconfig(conf)

db.test.insert({ count: 2 }, { writeConcern: { w: 3 } })
// → 等待 5s 才返回

db.test.insert({ count: 5 }, { writeConcern: { w: 3, wtimeout: 3000 } })
// → writeConcernError: waiting for replication timed out

// 主节点
db.test.insert({ count: 2 }, { writeConcern: { w: 3 } })  // 三副本确认
db.test.find({}).readPref("secondary")                    // 1 line

// 从节点 1、2
db.fsyncLock()                                             // 锁住写入

// 主节点
db.test.insert({ count: 3 })                              // 只写主
db.test.find({})                                          // 2 lines
db.test.find().readPref("secondary")                      // 1 line ← 从未同步

// 从节点 1、2
db.fsyncUnlock()
// 主节点
db.test.find().readPref("secondary")                      // 2 lines ← 已同步

// 从节点 1、2 锁定写入，模拟主从不一致
db.fsyncLock()
// 主节点
db.test.insert({ count: 3 })

db.test.find().readConcern("local")     // 可能脏读（若该写操作后被回滚）
db.test.find().readConcern("majority")  // 避免脏读

db.tx.drop();
db.tx.insertMany([{ x: 1 }, { x: 2 }]);

var session = db.getMongo().startSession();
session.startTransaction();
var coll = session.getDatabase('test').getCollection('tx');
coll.updateOne({ x: 2 }, { $set: { y: 1 } });

coll.find();    // 事务内：{x:1}, {x:2, y:1}
db.tx.find();   // 事务外：{x:1}, {x:2}             ← 隔离

session.commitTransaction();
db.tx.find();   // {x:1}, {x:2, y:1}

db.tx.drop();
db.tx.insertMany([{ x: 1 }, { x: 2 }]);

var session = db.getMongo().startSession();
session.startTransaction({
  readConcern: { level: "snapshot" },
  writeConcern: { w: "majority" }
});
var coll = session.getDatabase('test').getCollection('tx');
coll.findOne({ x: 1 });                  // 首次读

// 事务外的更新
db.tx.updateOne({ x: 1 }, { $set: { y: 1 } });

coll.findOne({ x: 1 });                  // 仍是首次的快照
session.abortTransaction();

// shell 1、2 同时开事务
session.startTransaction({
  readConcern: { level: "snapshot" },
  writeConcern: { w: "majority" }
});

// shell 1
coll.updateOne({ x: 1 }, { $set: { y: 1 } });   // ok

// shell 2
coll.updateOne({ x: 1 }, { $set: { y: 1 } });
// MongoServerError: WriteConflict — 须 retry 或 abort

# 1 配置域名解析
# 2 准备分片目录

# 3 创建第 1 个分片复制集 member1:27010 member3:27010 member5:27010
mongod --bind_ip_all --replSet shard1 --shardsvr --wiredTigerCacheSizeGB 1
# --shardsvr               标注为分片节点
# --wiredTigerCacheSizeGB  缓存大小

# 4 初始化分片复制集

# 5 创建 config server 复制集 member1:27019 member3:27019 member5:27019
mongod --bind_ip_all --replSet config --configsvr --wiredTigerCacheSizeGB 1
# --configsvr              标注为配置节点

# 6 初始化 config server 复制集

# 7 搭建 mongos
mongos --bind_ip_all --configdb config/member1:27019,member3:27019,member5:27019

# 连接 mongos 添加分片
sh.addShard("shard1/member1:27010,member3:27010,member5:27010")

# 8 创建分片表
sh.enableSharding("foo");
sh.shardCollection("foo.bar", {_id: "hashed"});
sh.status();

# 插入测试数据
use foo
for (var i = 0; i < 10000; i++) {
  db.bar.insert({ i: i })
}

# 9~11 创建并加入第 2 个分片复制集
sh.addShard("shard2/member2:27011,member4:27011,member6:27011")

db.serverStatus()              // 从上次启动到现在
db.serverStatus().opcounters
db.isMaster()
mongostat                      // CLI 工具

db.createRole({
   role: "readWriteRole",
   privileges: [{
      resource: { db: "myDatabase", collection: "sample" },
      actions: [ "find", "insert", "update", "remove" ]
   }],
   roles: [{ role: 'read', db: 'sampledb' }]
})

db.createUser({
  user: 'sampleUser',
  pwd: 'pwd',
  roles: [{ role: 'readWriteRole', db: 'admin' }]
})

mongod --noscripting   # 禁止脚本执行
mongod --auth          # 强制鉴权

db.member.createIndex({ city: 1 }, { background: true })

# vim host-path-pv-pvc.yml
apiVersion: v1
kind: PersistentVolume
metadata:
  # 只有 10MB 容量的存储设备
  name: host-10m-pv
spec:
  # kubectl explain PersistentVolume.spec.storageClassName
  storageClassName:
    host-vol
    # kubectl explain PersistentVolume.spec.accessModes
  accessModes:
    # ReadWriteOnce：存储卷可读可写，但只能被一个节点上的 Pod 挂载。
    # ReadOnlyMany：存储卷只读不可写，可以被任意节点上的 Pod 多次挂载。
    # ReadWriteMany：存储卷可读可写，也可以被任意节点上的 Pod 多次挂载。
    - ReadWriteOnce
  capacity:
    #  Ki/Mi/Gi
    storage: 10Mi
  hostPath:
    path: /tmp/host-10m-pv/

---
# PVC 不表示实际的存储，而是一个“申请”或者“声明”
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: host-5m-pvc
spec:
  storageClassName: host-vol
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Mi

mkdir /tmp/host-10m-pv
kubectl apply -f host-path-pv-pvc.yml
kubectl get pv
# NAME          CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                 STORAGECLASS   REASON   AGE
# host-10m-pv   10Mi       RWO            Retain           Bound    default/host-5m-pvc   host-vol                4s
kubectl get pvc
# NAME          STATUS   VOLUME        CAPACITY   ACCESS MODES   STORAGECLASS   AGE
# host-5m-pvc   Bound    host-10m-pv   10Mi       RWO            host-vol       6s

# vim host-pvc-pod.yml
apiVersion: v1
kind: Pod
metadata:
  name: host-pvc-pod
spec:
  volumes:
    - name: host-pvc-vol
      persistentVolumeClaim:
        claimName: host-5m-pvc
  containers:
    - name: ngx-pvc-pod
      image: nginx:alpine
      ports:
        - containerPort: 80
      volumeMounts:
        - name: host-pvc-vol
          mountPath: /tmp

kubectl apply -f host-pvc-pod.yml
kubectl get pod -o wide
kubectl exec -it host-pvc-pod -- sh
cd /tmp && touch a.md
# check in worker node
/tmp/host-10m-pv/

# 修改源 https://mirrors.cloud.tencent.com/
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.cloud.tencent.com/repo/centos7_base.repo
yum clean all
yum makecache

# install docker
yum remove docker \
            docker-client \
            docker-client-latest \
            docker-common \
            docker-latest \
            docker-latest-logrotate \
            docker-logrotate \
            docker-engine

yum install -y yum-utils

yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo

yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin

systemctl start docker
docker -v
docker run hello-world

# 改主机名
vi /etc/hostname
# reboot

# 把 cgroup 的驱动程序改成 systemd
# 使用 Docker 作为 Kubernetes 的底层支持
cat <<EOF | sudo tee /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

systemctl enable docker
systemctl daemon-reload
systemctl restart docker
docker version

# https://kubernetes.io/zh-cn/docs/setup/production-environment/container-runtimes/
# 转发 IPv4 并让 iptables 看到桥接流量
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# check
lsmod | grep br_netfilter
lsmod | grep overlay

# 设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

# 应用 sysctl 参数而不重新启动
sudo sysctl --system

sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
# net.bridge.bridge-nf-call-iptables = 1
# net.bridge.bridge-nf-call-ip6tables = 1
# net.ipv4.ip_forward = 1

# 关闭 Linux 的 swap 分区
swapoff -a
sed -ri '/\sswap\s/s/^#?/#/' /etc/fstab

# https://developer.aliyun.com/mirror/kubernetes
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.cloud.tencent.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
EOF

yum clean all
yum makecache

# 将 SELinux 设置为 permissive 模式（相当于将其禁用）
setenforce 0
sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

# 新版本搞不定
# yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes --nogpgcheck

yum --showduplicate list kubelet
yum install -y kubelet-1.23.16-0 kubeadm-1.23.16-0 kubectl-1.23.16-0 --disableexcludes=kubernetes --nogpgcheck

systemctl enable --now kubelet

kubeadm version
kubectl version --output=yaml
kubelet --version

# kubeadm config images list
kubeadm config images list --kubernetes-version v1.23.16

vim /etc/containerd/config.toml
#disabled_plugins = ["cri"]

systemctl enable containerd
systemctl restart containerd
systemctl status containerd

systemctl enable kubelet.service
systemctl restart kubelet
systemctl status kubelet

containerd config default > /etc/containerd/config.toml

yum install -y nc
nc 127.0.0.1 6443

kubeadm init -h
kubeadm reset -f
rm -rf ~/.kube/config

kubeadm init \
    --image-repository=registry.aliyuncs.com/google_containers \
    --pod-network-cidr=10.10.0.0/16 \
    --kubernetes-version=v1.23.16 \
    --v=9

# Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
#   https://kubernetes.io/docs/concepts/cluster-administration/addons/

# success
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

export KUBECONFIG=/etc/kubernetes/admin.conf

kubectl get node
# NAME     STATUS     ROLES                  AGE    VERSION
# master   NotReady   control-plane,master   2m4s   v1.23.16

# dubug
systemctl restart docker
systemctl restart kubelet
systemctl restart containerd
journalctl -xeu kubelet
crictr ps -a
crictl --runtime-endpoint unix:///var/run/containerd/containerd.sock ps -a | grep kube | grep -v pause

kubectl get pods -n kube-system
kubectl describe pods -n kube-system

# Flannel 网络插件 https://github.com/flannel-io/flannel/tree/master
# curl https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml --output kube-flannel.yml
#  net-conf.json: |
#     {
#       "Network": "10.10.0.0/16",
#       "Backend": {
#         "Type": "vxlan"
#       }
#     }
kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml
kubectl get node
# NAME     STATUS   ROLES                  AGE   VERSION
# master   Ready    control-plane,master   14h   v1.23.16

# show join command in control-plane
kubeadm token create --print-join-command
# work join; 云服务记得开放入站端口
telnet 172.21.0.5 6443
systemctl enable kubelet.service
kubeadm join 172.21.0.5:6443 --token xxx --discovery-token-ca-cert-hash sha256:xxx --v=9
# check in control-plane
kubectl get nodes
# NAME            STATUS   ROLES                  AGE     VERSION
# master          Ready    control-plane,master   14h     v1.23.16
# vm-0-9-centos   Ready    <none>                 3m27s   v1.23.16

# run nginx
kubectl run ngx --image=nginx:alpine
kubectl get pod -o wide
# NAME   READY   STATUS    RESTARTS   AGE   IP          NODE      NOMINATED NODE   READINESS GATES
# ngx    1/1     Running   0          52m   10.10.1.2   woker01   <none>           <none>

kubectl api-resources

export out="--dry-run=client -o yaml"
kubectl create deploy ngx-dep --image=nginx:alpine $out > ngx-dep.yml
vim ngx-dep.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  creationTimestamp: null
  labels:
    app: ngx-dep
  name: ngx-dep
spec:
  # 定义了 Pod 的“期望数量”，Kubernetes 会自动维护 Pod 数量到正常水平
  replicas: 2
  # 定义了基于 labels 筛选 Pod 的规则，它必须与 template 里 Pod 的 labels 一致
  selector:
    matchLabels:
      app: ngx-dep
  strategy: {}
  template:
    metadata:
      creationTimestamp: null
      # 贴标签
      labels:
        app: ngx-dep
    spec:
      containers:
        - image: nginx:alpine
          name: nginx
          resources: {}
status: {}

# replicas: 2
kubectl apply -f ngx-dep.yml

kubectl get deploy
# NAME      READY   UP-TO-DATE   AVAILABLE   AGE
# ngx-dep   2/2     2            2           57s
kubectl get pod
# NAME                      READY   STATUS    RESTARTS   AGE
# ngx-dep-bfbb5f64b-96scb   1/1     Running   0          3m20s
# ngx-dep-bfbb5f64b-qnzbh   1/1     Running   0          3m20s

# 测试自启恢复
kubectl delete pod ngx-dep-bfbb5f64b-qnzbh
kubectl get pod
# NAME                      READY   STATUS    RESTARTS   AGE
# ngx-dep-bfbb5f64b-7n724   1/1     Running   0          33s
# ngx-dep-bfbb5f64b-96scb   1/1     Running   0          4m52s

# 测试伸缩
kubectl scale --replicas=5 deploy ngx-dep
kubectl get pod
# NAME                      READY   STATUS    RESTARTS   AGE
# ngx-dep-bfbb5f64b-7n724   1/1     Running   0          77s
# ngx-dep-bfbb5f64b-7xhbs   1/1     Running   0          7s
# ngx-dep-bfbb5f64b-96scb   1/1     Running   0          5m36s
# ngx-dep-bfbb5f64b-97qp5   1/1     Running   0          7s
# ngx-dep-bfbb5f64b-vjn4q   1/1     Running   0          7s

# 筛选标签 ==、!=、in、notin
kubectl get pod -l app=nginx
kubectl get pod -l 'app in (ngx, nginx, ngx-dep)'

kubectl api-resources

# export out="--dry-run=client -o yaml"
# kubectl create deploy redis-ds --image=redis:5-alpine $out
# kind modify DaemonSet, delete spec.replicas
# https://kubernetes.io/zh-cn/docs/concepts/workloads/controllers/daemonset/
# vim redis-ds.yml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: redis-ds
  labels:
    app: redis-ds
spec:
  # 和 deplayment 比没有 replicas
  selector:
    matchLabels:
      name: redis-ds
  template:
    metadata:
      labels:
        name: redis-ds
    spec:
      containers:
        - image: redis:5-alpine
          name: redis
          ports:
            - containerPort: 6379

kubectl apply -f redis-ds.yml
kubectl get ds
# NAME       DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
# redis-ds   2         2         2       2            2           <none>          30m
kubectl get pod -o wide
# 两个 worker 的场景
# NAME                      READY   STATUS    RESTARTS   AGE   IP           NODE      NOMINATED NODE   READINESS GATES
# redis-ds-9r96k            1/1     Running   0          2m52s   10.10.3.2    woker02   <none>           <none>
# redis-ds-hdl28            1/1     Running   0          21m     10.10.1.11   woker01   <none>           <none>
# Master 节点却被排除在外了

kubectl describe node master
# Taints:             node-role.kubernetes.io/master:NoSchedule
# 污点会拒绝 Pod 调度到本节点上运行
kubectl describe node woker01
# Taints:             <none>

# - 出掉 master 污点
kubectl taint node master node-role.kubernetes.io/master:NoSchedule-
# NAME       DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
# redis-ds   3         3         3       3            3           <none>          31m

# Pod 添加 tolerations
# kubectl explain ds.spec.template.spec.tolerations
# vim redis-ds-t.yml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: redis-ds-t
  labels:
    app: redis-ds-t
spec:
  # 和 deplayment 比没有 replicas
  selector:
    matchLabels:
      name: redis-ds-t
  template:
    metadata:
      labels:
        name: redis-ds-t
    spec:
      containers:
        - image: redis:5-alpine
          name: redis
          ports:
            - containerPort: 6379
      # 容忍 node-role.kubernetes.io/master
      tolerations:
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
          operator: Exists

kubectl apply -f redis-ds-t.yml
kubectl get ds
# NAME         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
# redis-ds     2         2         2       2            2           <none>          41m
# redis-ds-t   3         3         3       3            3           <none>          6s
# 差别在 master
kubectl get pod -o wide
# NAME                      READY   STATUS    RESTARTS   AGE   IP           NODE      NOMINATED NODE   READINESS GATES
# redis-ds-9r96k            1/1     Running   0          23m   10.10.3.2    woker02   <none>           <none>
# redis-ds-hdl28            1/1     Running   0          42m   10.10.1.11   woker01   <none>           <none>
# redis-ds-t-4mptv          1/1     Running   0          80s   10.10.3.4    woker02   <none>           <none>
# redis-ds-t-dpcl8          1/1     Running   0          80s   10.10.1.12   woker01   <none>           <none>
# redis-ds-t-kdjmn          1/1     Running   0          80s   10.10.0.6    master    <none>           <none>

ll -a /etc/kubernetes/manifests
# -rw------- 1 root root 2274 Feb 22 12:47 etcd.yaml
# -rw------- 1 root root 3358 Feb 22 12:47 kube-apiserver.yaml
# -rw------- 1 root root 2878 Feb 22 12:47 kube-controller-manager.yaml
# -rw------- 1 root root 1465 Feb 22 12:47 kube-scheduler.yaml

# flannel 就是一个 DaemonSet
kubectl get ns
kubectl get ds -n kube-flannel
# NAME              DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
# kube-flannel-ds   3         3         3       3            3           <none>          3h54m

export out="--dry-run=client -o yaml"
kubectl expose deploy ngx-dep --port=80 --target-port=80 $ou

# vim ngx-svc.yml
apiVersion: v1
kind: Service
metadata:
  name: ngx-svc
spec:
  ports:
    - port: 80
      protocol: TCP
      targetPort: 80
  selector:
    app: ngx-dep
status:
  loadBalancer: {}

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: ngx-conf
data:
  default.conf: |
    server {
      listen 80;
      location / {
        default_type text/plain;
        return 200
          'srv : $server_addr:$server_port\nhost: $hostname\nuri : $request_method $host $request_uri\ndate: $time_iso8601\n';
      }
    }

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ngx-dep
spec:
  replicas: 2
  selector:
    matchLabels:
      app: ngx-dep
  template:
    metadata:
      labels:
        app: ngx-dep
    spec:
      volumes:
        - name: ngx-conf-vol
          configMap:
            name: ngx-conf
      containers:
        - image: nginx:alpine
          name: nginx
          ports:
            - containerPort: 80
          volumeMounts:
            - mountPath: /etc/nginx/conf.d
              name: ngx-conf-vol

kubectl apply -f ngx-svc.yml
kubectl get svc
# NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)   AGE
# kubernetes   ClusterIP   10.96.0.1        <none>        443/TCP   5h10m
# ngx-svc      ClusterIP   10.109.131.132   <none>        80/TCP    35s
# 虚地址 10.109.131.132
kubectl describe svc ngx-svc
# Name:              ngx-svc
# Namespace:         default
# Labels:            <none>
# Annotations:       <none>
# Selector:          app=ngx-dep
# Type:              ClusterIP
# IP Family Policy:  SingleStack
# IP Families:       IPv4
# IP:                10.109.131.132
# IPs:               10.109.131.132
# Port:              <unset>  80/TCP
# TargetPort:        80/TCP
# Endpoints:         10.10.1.13:80,10.10.3.5:80
# Session Affinity:  None
# Events:            <none>
kubectl get pod -o wide
NAME                       READY   STATUS    RESTARTS   AGE    IP           NODE
ngx-dep-6796688696-cwm8f   1/1     Running   0          2m4s   10.10.3.5    woker02
ngx-dep-6796688696-khjnv   1/1     Running   0          2m2s   10.10.1.13   woker01
# same Endpoints

# 因为 Service、 Pod 的 IP 地址都是 Kubernetes 集群的内部网段
#   所以我们需要用 kubectl exec 进入到 Pod 内部
kubectl exec -it ngx-dep-6796688696-cwm8f -- sh
curl 10.109.131.132
# srv : 10.10.3.5:80
# host: ngx-dep-6796688696-cwm8f
# uri : GET 10.109.131.132 /
# date: 2023-02-22T10:09:49+00:00
curl 10.109.131.132
# srv : 10.10.1.13:80
# host: ngx-dep-6796688696-khjnv
# uri : GET 10.109.131.132 /
# date: 2023-02-22T10:09:50+00:00

# 测试恢复
kubectl delete pod ngx-dep-6796688696-khjnv
kubectl describe svc ngx-svc
# Endpoints:         10.10.1.14:80,10.10.3.5:80
# 之前是 10.10.1.13:80,10.10.3.5:80

# 测试扩容
kubectl scale --replicas=5 deploy ngx-dep
kubectl describe svc ngx-svc
# Endpoints:         10.10.1.14:80,10.10.1.15:80,10.10.3.5:80 + 2 more...

# Name:              ngx-svc
# Namespace:         default
kubectl exec -it ngx-dep-6796688696-cwm8f -- sh
curl ngx-svc
# srv : 10.10.3.5:80
# host: ngx-dep-6796688696-cwm8f
# uri : GET ngx-svc /
# date: 2023-02-22T10:19:25+00:00
curl ngx-svc.default
# srv : 10.10.3.6:80
# host: ngx-dep-6796688696-lpcfs
# uri : GET ngx-svc.default /
# date: 2023-02-22T10:19:41+00:00
curl ngx-svc.default.svc.cluster.local
# srv : 10.10.3.5:80
# host: ngx-dep-6796688696-cwm8f
# uri : GET ngx-svc.default.svc.cluster.local /
# date: 2023-02-22T10:20:04+00:00

# kubectl explain svc.spec.type
# vim ngx-svc.yml
  ...
  type: NodePort

kubectl apply -f ngx-svc.yml
kubectl get svc
# NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE
# kubernetes   ClusterIP   10.96.0.1        <none>        443/TCP        5h56m
# ngx-svc      NodePort    10.109.131.132   <none>        80:30916/TCP   46m
# Service 的默认类型是“ClusterIP”，只能在集群内部访问，
#   如果改成“NodePort”，就会在节点上开启一个随机端口号，让外界也能够访问内部的服务。
curl localhost:30916
# srv : 10.10.1.15:80
# host: ngx-dep-6796688696-l6skl
# uri : GET localhost /
# date: 2023-02-22T10:48:32+00:00

export out="--dry-run=client -o yaml"
kubectl create ing ngx-ing --rule="ngx.test/=ngx-svc:80" --class=ngx-ink $out

# vim ingress.yml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  creationTimestamp: null
  name: ngx-ing
spec:
  ingressClassName: ngx-ink
  rules:
    - host: ngx.test
      http:
        # 路径的匹配方式
        paths:
          - backend:
              service:
                name: ngx-svc
                port:
                  number: 80
            path: /
            # 精确匹配（Exact）或前缀匹配（Prefix）
            pathType: Exact
status:
  loadBalancer: {}

---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
  name: ngx-ink
spec:
  controller: nginx.org/ingress-controller

kubectl apply -f ingress.yml
# NAME      CONTROLLER                     PARAMETERS   AGE
# ngx-ink   nginx.org/ingress-controller   <none>       15s
kubectl get ing
# NAME      CLASS     HOSTS      ADDRESS   PORTS   AGE
# ngx-ing   ngx-ink   ngx.test             80      84s
kubectl describe ing ngx-ing
# Name:             ngx-ing
# Labels:           <none>
# Namespace:        default
# Address:
# Default backend:  default-http-backend:80 (<error: endpoints "default-http-backend" not found>)
# Rules:
#   Host        Path  Backends
#   ----        ----  --------
#   ngx.test
#               /   ngx-svc:80 (10.10.1.14:80,10.10.1.15:80)
# Annotations:  <none>
# Events:       <none>

git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.0.2
cd kubernetes-ingress/deployments

# Configure RBAC
# 为Ingress控制器创建一个命名空间和一个服务账户
kubectl apply -f common/ns-and-sa.yaml
# 为服务账户创建一个集群角色和集群角色绑定
kubectl apply -f rbac/rbac.yaml

# Create Common Resources
# 为NGINX的默认服务器创建一个带有TLS证书和密钥的秘密
kubectl apply -f common/default-server-secret.yaml
# 创建一个 config map，用于定制NGINX配置。
kubectl apply -f common/nginx-config.yaml
# 创建一个IngressClass资源
kubectl apply -f common/ingress-class.yaml

# Create Custom Resources
# kubectl apply -f common/crds/
vim deployment/nginx-ingress.yaml
# args add:
# -enable-custom-resources=false

# Run the Ingress Controller
kubectl apply -f deployment/nginx-ingress.yaml

# check
kubectl get pods --namespace=nginx-ingress
# NAME                             READY   STATUS    RESTARTS   AGE
# nginx-ingress-5f98f8f5f9-nnkv7   1/1     Running   0          3m14s

# Get Access to the Ingress Controller
kubectl create -f service/nodeport.yaml
kubectl get service -n nginx-ingress
# NAME            TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)                      AGE
# nginx-ingress   NodePort   10.111.210.52   <none>        80:31754/TCP,443:30188/TCP   5s

# debug
kubectl get IngressClass
kubectl get ing -n nginx-ingress
kubectl get deploy -n nginx-ingress
kubectl get pod -n nginx-ingress -o wide

kubectl describe service -n nginx-ingress -o wide
kubectl describe pod -n nginx-ingress

# 命令kubectl port-forward，它可以直接把本地的端口映射到 Kubernetes 集群的某个 Pod 里
kubectl port-forward -n nginx-ingress nginx-ingress-5f98f8f5f9-nnkv7 8080:80 &

# vim wp-maria.yml
apiVersion: v1
kind: ConfigMap
metadata:
  name: maria-cm
data:
  DATABASE: "db"
  USER: "wp"
  PASSWORD: "123"
  ROOT_PASSWORD: "123"

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: maria-dep
  name: maria-dep
spec:
  replicas: 1
  selector:
    matchLabels:
      app: maria-dep
  template:
    metadata:
      labels:
        app: maria-dep
    spec:
      containers:
        - image: mariadb:10
          name: mariadb
          ports:
            - containerPort: 3306
          envFrom:
            - prefix: "MARIADB_"
              configMapRef:
                name: maria-cm

---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: maria-dep
  name: maria-svc
spec:
  ports:
    - port: 3306
      protocol: TCP
      targetPort: 3306
  selector:
    app: maria-dep

kubectl apply -f wp-maria.yml
kubectl get pod
kubectl get deploy
kubectl get svc

# vim wp-app.yml
apiVersion: v1
kind: ConfigMap
metadata:
  name: wp-cm
data:
  # DNS HOST
  HOST: "maria-svc"
  USER: "wp"
  PASSWORD: "123"
  NAME: "db"

---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: wp-dep
  name: wp-dep
spec:
  replicas: 2
  selector:
    matchLabels:
      app: wp-dep
  template:
    metadata:
      labels:
        app: wp-dep
    spec:
      containers:
        - image: wordpress:5
          name: wordpress
          ports:
            - containerPort: 80
          envFrom:
            - prefix: "WORDPRESS_DB_"
              configMapRef:
                name: wp-cm

---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: wp-dep
  name: wp-svc
spec:
  ports:
    - name: http80
      port: 80
      protocol: TCP
      targetPort: 80
      # 指定端口
      nodePort: 30088
  selector:
    app: wp-dep
  # NodePort
  type: NodePort