Commit 737dd517 authored by Dmytro Bogatov's avatar Dmytro Bogatov 💕

Move to DO K8S Limited Availability. Deploy Yara2.

parent 9e7606ec
Pipeline #4091 failed with stage
in 52 seconds
......@@ -13,7 +13,7 @@ infra/sources/shevastream/appsettings.json
infra/sources/do-volume-provisioner/do-secret.yaml
infra/sources/gitlab-runner/config.yaml
infra/terraform/spaces/
infra/spaces/
infra/spaces-buffer
......
......@@ -4,7 +4,7 @@ metadata:
name: __NAME__
namespace: review
annotations:
kubernetes.io/ingress.class: "public"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/auth-secret: __NAME__-basic-auth
......
......@@ -9,6 +9,7 @@ cd "${0%/*}"
CWD=$(pwd)
REPLICAS="3"
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
source .secret.sh
......@@ -183,9 +184,9 @@ else
rm -rf dashboard/
mkdir -p dashboard
cp ./sources/dashboard/*.yaml ./dashboard
cp ./sources/dashboard/ingress.yaml ./dashboard
DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}"
sed -i -e "s#__DASHBOARD_TOKEN__#$DASHBOARD_TOKEN#g" dashboard/ingress.yaml
......
#!/usr/bin/env bash
set -e
shopt -s globstar
# Ensure that the CWD is set to script's location
cd "${0%/*}"
CWD=$(pwd)
set -x
SWAPFILE="/var/vm/swapfile1"
if [[ $EUID -ne 0 ]];
then
echo "This script must be run as root"
exit 1
fi
if ! [ $# -eq 1 ]
then
echo "APIKEY missing"
exit 1
fi
# echo "Producing SWAP config"
# cat >/etc/systemd/system/var-vm-swapfile1.swap <<SWAP
# [Unit]
# Description=Turn on swapcd clu
# [Swap]
# What=/var/vm/swapfile1
# [Install]
# WantedBy=multi-user.target
# SWAP
# echo "Adding SWAP"
# if [ -f $SWAPFILE ];
# then
# echo "File $SWAPFILE already exists"
# else
# mkdir -p /var/vm
# fallocate -l 2048m /var/vm/swapfile1
# chmod 600 /var/vm/swapfile1
# mkswap /var/vm/swapfile1
# systemctl enable --now var-vm-swapfile1.swap
# fi
# echo "Enabling SWAP support for kubelet"
# if grep -q "fail-swap-on=false" "/etc/systemd/system/kubelet.service";
# then
# echo "Already enabled"
# else
# sed -i '/kubelet-wrapper/a \ --fail-swap-on=false \\\' /etc/systemd/system/kubelet.service # TODO
# fi
echo "Installing status site daemon"
NAME=$(cut -d " " -f 3 <<< $(hostnamectl | head -n 1))
APIKEY=$1
cat >/home/core/report-cpu-load.sh <<CPULOAD
#!/usr/bin/env bash
set -e
CPU_LOAD=\$(top -b -n2 -p 1 | fgrep "Cpu(s)" | tail -1 | awk -F'id,' -v prefix="$prefix" '{ split(\$1, vs, ","); v=vs[length(vs)]; sub("%", "", v); printf "%s%d", prefix, 100 - v }')
curl -X POST -s "https://status.dbogatov.org/api/cpuload" --data "value=\${CPU_LOAD}&source=$NAME" --header "apikey: $APIKEY" > /dev/null
CPULOAD
chmod +x /home/core/report-cpu-load.sh
cat >/etc/systemd/system/cpu-load.service <<CRONSERVICE
[Unit]
Description=Reports CPU load to status site
[Service]
Type=oneshot
ExecStart=/bin/bash -c '/home/core/report-cpu-load.sh'
CRONSERVICE
cat >/etc/systemd/system/cpu-load.timer <<CRONTIMER
[Unit]
Description=Run cpu-load.service every minute
[Timer]
OnCalendar=*:0/1
CRONTIMER
systemctl daemon-reload
systemctl start cpu-load.timer
echo "Done."
......@@ -34,91 +34,41 @@ then
fi
# DOCKERPASS=...
# EMAIL=...
# PASSWORD=...
source .secret.sh
CERTDIRPATH=$1
NAME=$2
KEBEFILE=""
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
STATUSSITECONFIG=$CERTDIRPATH/appsettings.production.yml
VERSION="1.12.7-do.1"
APIKEY=$(cat $STATUSSITECONFIG | grep "ApiKey:" | cut -d'"' -f 2)
if [ "$KUBEOLDWAY" = true ] ; then
# Initiate cluster
echo "Initializing cluster on DigitalOcean"
# Add identity
ssh-add ~/.ssh/id_rsa
cd $CWD/terraform/clusters/
terraform destroy -force || true # might be that there is nothing to destroy
my-sleep 30
terraform init
terraform apply -auto-approve
my-sleep 60
# Add SWAP to master
echo "Adding SWAP file to the nodes"
cd "$CWD"
IPS=("$(dig @ns1.digitalocean.com +short A alice-workers.$NAME.dbogatov.org)")
IPS+=("$(dig @ns1.digitalocean.com +short A alice.$NAME.dbogatov.org)")
for ip in ${IPS[@]}
do
echo "Executing init script for node $ip"
scp -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" node-init.sh core@$ip:/home/core
docker info > /dev/null
ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" core@$ip "sudo /home/core/node-init.sh $APIKEY"
APITOKEN=$(cat ~/.config/digital-ocean/token)
done
# PROVISION
my-sleep 30
else
APITOKEN=$(cat ~/.config/digital-ocean/token)
CLUSTER_ID=$(curl -s -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" -d '{"name": "'"$NAME"'","region": "nyc1","version": "'"$VERSION"'","node_pools": [{"size": "s-1vcpu-2gb","count": 4,"name": "main-pool"}]}' "https://api.digitalocean.com/v2/kubernetes/clusters" | jq -r '.kubernetes_cluster.id')
echo "Cluster ID: $CLUSTER_ID"
STATE="init"
while [ "$STATE" != "running" ]
do
echo "Current state is $STATE"
my-sleep 10
STATE=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID" | jq -r '.kubernetes_cluster.status.state')
done
echo "State is $STATE"
curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID/kubeconfig" > kubeconfig.yaml
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
echo "Dowloaded kubefile.yaml"
EXTERNALIPS=$(kubectl $KEBEFILE get nodes -o jsonpath='{ $.items[*].status.addresses[?(@.type=="ExternalIP")].address }')
IFS=' ' read -r -a IPS <<< "$EXTERNALIPS"
curl -X DELETE --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/workers.dbogatov.org/A
for ip in "${IPS[@]}"
do
echo "Adding $ip"
curl -X POST -d "$ip" --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/workers.dbogatov.org/A
done
echo "Cluster provisioned!"
fi
CLUSTER_ID=$(curl -s -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" -d '{"name": "'"$NAME"'","region": "nyc1","version": "'"$VERSION"'","node_pools": [{"size": "s-1vcpu-2gb","count": 4,"name": "main-pool"}]}' "https://api.digitalocean.com/v2/kubernetes/clusters" | jq -r '.kubernetes_cluster.id')
echo "Cluster ID: $CLUSTER_ID"
STATE="init"
while [ "$STATE" != "running" ]
do
echo "Current state is $STATE"
my-sleep 10
STATE=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID" | jq -r '.kubernetes_cluster.status.state')
done
echo "State is $STATE"
exit
curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID/kubeconfig" > kubeconfig.yaml
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
echo "Dowloaded kubefile.yaml"
docker info > /dev/null
echo "Cluster provisioned!"
cd "$CWD"
# NAMESPACES
echo "Creating namespaces and saving SSL certs"
......@@ -136,21 +86,15 @@ done
kubectl $KEBEFILE create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist
# Deploy addons
echo "Deploying addons"
cd $CWD/terraform
# RESOURCES
echo "Deploying dashboard"
kubectl $KEBEFILE apply -R -f addons/dashboard/
kubectl $KEBEFILE apply -R -f ./sources/dashboard/all.yaml
echo "Deploying NGINX Ingress"
kubectl $KEBEFILE apply -R -f addons/nginx-ingress/digital-ocean/
cd $CWD
kubectl $KEBEFILE apply -R -f ./sources/nginx/mandatory.yaml
echo "Deploying DO volume provisioner"
......@@ -163,7 +107,7 @@ kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/di
echo "Deploying websites' settings"
kubectl $KEBEFILE create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG
kubectl $KEBEFILE create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG || true # may exist
kubectl $KEBEFILE create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json
echo "Generating config files"
......@@ -176,7 +120,7 @@ kubectl $KEBEFILE apply -R -f sources/nginx/
kubectl $KEBEFILE apply -R -f services/
kubectl $KEBEFILE apply -R -f ./dashboard/
kubectl $KEBEFILE apply -R -f ./dashboard/ingress.yaml
echo "Deploying status site"
......
......@@ -7,29 +7,90 @@ set -e
source sources/data.sh
source .secret.sh
for domain in "${!DOMAINS[@]}"
do
echo "Setting $domain..."
VALUES=($(dig +short A ${AVALUE}))
echo "Values are ${VALUES[*]}"
VALUES=($(dig +short A ${DOMAINS[${domain}]}))
#
# $1 - domain
#
update-domain () {
domain=$1
variants=($domain *.$domain www.$domain)
echo "Setting $domain..."
echo "Removing A records for $domain, *.$domain and www.$domain"
for variant in ${variants[@]}
do
while true
do
result=$(curl -s -X DELETE --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/$variant/A)
if [[ $result = *"error"* ]]
then
echo "Error clearing $variant. Retrying..."
sleep 3
else
echo "Successfully cleared $variant"
break
fi
done
for value in ${VALUES[@]}
do
echo "Setting A record value $value for $domain, *.$domain and www.$domain"
curl -X DELETE --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/$domain/A
curl -X DELETE --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/www.$domain/A
curl -X DELETE --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/*.$domain/A
while true
do
result=$(curl -s -X POST -d "$value" --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/$variant/A)
if [[ $result = *"error"* ]]
then
echo "Error setting $variant to $value. Retrying..."
sleep 3
else
echo "Successfully set $variant to $value"
break
fi
done
for value in ${VALUES[@]}
do
done
done
}
echo "Setting A record value $value for $domain, *.$domain and www.$domain"
for domain in "${!DOMAINS[@]}"
do
update-domain $domain
done
curl -X POST -d "$value" --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/$domain/A
curl -X POST -d "$value" --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/www.$domain/A
curl -X POST -d "$value" --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/*.$domain/A
FAIL=false
while true
do
for domain in "${!DOMAINS[@]}"
do
variants=($domain *.$domain www.$domain)
for variant in ${variants[@]}
do
record=$(dig +short A ${variant})
if [ "$record" == "${VALUES[0]}" ]
then
echo "PASS"
else
echo "FAIL for $variant"
FAIL=true
update-domain $domain
fi
done
done
if [ "$FIAL" == true ]
then
echo "Last test run failed, so rerun tests"
continue
fi
break
done
echo "Done!"
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "public"
nginx.ingress.kubernetes.io/auth-signin: https://$host/oauth2/start
nginx.ingress.kubernetes.io/auth-url: https://$host/oauth2/auth
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
nginx.ingress.kubernetes.io/configuration-snippet: |
proxy_set_header Authorization "Bearer __DASHBOARD_TOKEN__";
name: external-auth-oauth2
namespace: kube-system
spec:
rules:
- host: dashboard-dbogatov-org.cluster.dbogatov.org
http:
paths:
- backend:
serviceName: kubernetes-dashboard
servicePort: 443
path: /
- host: dashboard.dbogatov.org
http:
paths:
- backend:
serviceName: kubernetes-dashboard
servicePort: 443
path: /
tls:
- hosts:
- dashboard-dbogatov-org.cluster.dbogatov.org
- dashboard.dbogatov.org
secretName: lets-encrypt
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: oauth2-proxy
namespace: kube-system
annotations:
kubernetes.io/ingress.class: "public"
spec:
rules:
- host: dashboard-dbogatov-org.cluster.dbogatov.org
http:
paths:
- backend:
serviceName: oauth2-proxy
servicePort: 4180
path: /oauth2
- host: dashboard.dbogatov.org
http:
paths:
- backend:
serviceName: oauth2-proxy
servicePort: 4180
path: /oauth2
tls:
- hosts:
- dashboard-dbogatov-org.cluster.dbogatov.org
- dashboard.dbogatov.org
secretName: lets-encrypt
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
k8s-app: oauth2-proxy
name: oauth2-proxy
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
k8s-app: oauth2-proxy
template:
metadata:
labels:
k8s-app: oauth2-proxy
spec:
containers:
- args:
- --provider=gitlab
- -login-url=https://git.dbogatov.org/oauth/authorize
- -redeem-url=https://git.dbogatov.org/oauth/token
- -validate-url=https://git.dbogatov.org/api/v4/user
- --email-domain=*
- --upstream=file:///dev/null
- --http-address=0.0.0.0:4180
# Register a new application
# https://github.com/settings/applications/new
env:
- name: OAUTH2_PROXY_CLIENT_ID
value: 558e2988459e785811749d3a1e2ea16f451cdd25bf033d074433c60a16ba878f
- name: OAUTH2_PROXY_CLIENT_SECRET
value: __OAUTH2_PROXY_CLIENT_SECRET__
# python -c 'import os,base64; print base64.b64encode(os.urandom(16))'
- name: OAUTH2_PROXY_COOKIE_SECRET
value: __OAUTH2_PROXY_COOKIE_SECRET__
image: docker.io/colemickens/oauth2_proxy:latest
imagePullPolicy: Always
name: oauth2-proxy
ports:
- containerPort: 4180
protocol: TCP
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: oauth2-proxy
name: oauth2-proxy
namespace: kube-system
spec:
ports:
- name: http
port: 4180
protocol: TCP
targetPort: 4180
selector:
k8s-app: oauth2-proxy
......@@ -160,3 +160,28 @@ spec:
targetPort: 8443
selector:
k8s-app: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kube-system
---
......@@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "public"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
nginx.ingress.kubernetes.io/auth-secret: basic-auth
nginx.ingress.kubernetes.io/auth-realm: "Authentication Required!"
......
......@@ -40,8 +40,8 @@ SERVICES["moon-travel-com-ua"]="registry.dbogatov.org/dbogatov/nginx-proxies/moo
SERVICES["vleskniga-com"]="registry.dbogatov.org/slokot/vleskniga:latest"
SERVICES["veles-russia-com"]="registry.dbogatov.org/slokot/veles-russia:latest"
SERVICES["photobarrat-com"]="registry.dbogatov.org/dbogatov/nginx-proxies/photobarrat-com:latest"
SERVICES["new-photobarrat-com"]="registry.dbogatov.org/dbogatov/new-photobarrat-com:latest"
# SERVICES["photobarrat-com"]="registry.dbogatov.org/dbogatov/nginx-proxies/photobarrat-com:latest"
# SERVICES["new-photobarrat-com"]="registry.dbogatov.org/dbogatov/new-photobarrat-com:latest"
SERVICES["bogatov-kiev-ua"]="registry.dbogatov.org/daddy/bogatov-kiev-ua:latest"
SERVICES["blog-bogatov-kiev-ua"]="registry.dbogatov.org/daddy/blog-bogatov-kiev-ua:latest"
......@@ -63,10 +63,10 @@ DOMAINS["bogatov.kiev.ua"]=$AVALUE
DOMAINS["darinagulley.com"]=$AVALUE
DOMAINS["moon-travel.com.ua"]=$AVALUE
DOMAINS["nigmatullina.org"]=$AVALUE
DOMAINS["photobarrat.com"]=$AVALUE
# DOMAINS["photobarrat.com"]=$AVALUE
DOMAINS["res-public.net"]=$AVALUE
DOMAINS["shevastream.com"]=$AVALUE
DOMAINS["travelus.com.ua"]=$AVALUE
# DOMAINS["travelus.com.ua"]=$AVALUE
DOMAINS["veles-russia.com"]=$AVALUE
DOMAINS["visajapan.com.ua"]=$AVALUE
DOMAINS["visasupport.com.ua"]=$AVALUE
......
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: grafana-dbogatov-org
namespace: monitoring
annotations:
kubernetes.io/ingress.class: "public"
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/from-to-www-redirect: "true"
spec:
tls:
- hosts:
- grafana.dbogatov.org
- grafana-dbogatov-org.cluster.dbogatov.org
secretName: lets-encrypt
rules:
- host: "grafana.dbogatov.org"
http:
paths:
- path: /
backend:
serviceName: grafana
servicePort: 80
- host: "grafana-dbogatov-org.cluster.dbogatov.org"
http:
paths:
- path: /
backend:
serviceName: grafana
servicePort: 80
......@@ -14,6 +14,7 @@ data:
proxy-set-headers: "ingress-nginx/custom-headers"
add-headers: "ingress-nginx/custom-headers"
server-name-hash-bucket-size: "1024"
# use-proxy-protocol: "true"
metadata:
name: nginx-configuration
namespace: ingress-nginx
......
......@@ -4,7 +4,7 @@ metadata:
name: status-dbogatov-org
namespace: status-site
annotations:
kubernetes.io/ingress.class: "public"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/from-to-www-redirect: "true"
......
apiVersion: v1
kind: Namespace
metadata:
name: reboot-coordinator
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: reboot-coordinator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: reboot-coordinator
subjects:
- kind: ServiceAccount
namespace: reboot-coordinator
name: default
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: reboot-coordinator
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- get
- update
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- watch
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- delete