Commit 9e7606ec authored by Dmytro Bogatov's avatar Dmytro Bogatov 💕

Update to DO K8S.

parent 75a4bb3b
......@@ -5,6 +5,7 @@
*.lock*
infra/services/
infra/dashboard/
infra/kubeconfig.yaml
.secret.sh
......
......@@ -11,7 +11,7 @@ CWD=$(pwd)
my-sleep () {
secs=$1
while [ $secs -gt 0 ]; do
echo -ne "Sleeping $secs\033[0K\r"
echo -ne "Waiting $secs\033[0K\r"
sleep 1
: $((secs--))
done
......@@ -38,51 +38,85 @@ source .secret.sh
CERTDIRPATH=$1
NAME=$2
KEBEFILE=""
STATUSSITECONFIG=$CERTDIRPATH/appsettings.production.yml
VERSION="1.12.7-do.1"
APIKEY=$(cat $STATUSSITECONFIG | grep "ApiKey:" | cut -d'"' -f 2)
# Initiate cluster
if [ "$KUBEOLDWAY" = true ] ; then
# Initiate cluster
echo "Initializing cluster on DigitalOcean"
echo "Initializing cluster on DigitalOcean"
# Add identity
ssh-add ~/.ssh/id_rsa
# Add identity
ssh-add ~/.ssh/id_rsa
cd $CWD/terraform/clusters/
terraform destroy -force || true # might be that there is nothing to destroy
cd $CWD/terraform/clusters/
terraform destroy -force || true # might be that there is nothing to destroy
my-sleep 30
my-sleep 30
terraform init
terraform apply -auto-approve
terraform init
terraform apply -auto-approve
my-sleep 60
my-sleep 60
# Add SWAP to master
# Add SWAP to master
echo "Adding SWAP file to the nodes"
echo "Adding SWAP file to the nodes"
cd "$CWD"
cd "$CWD"
IPS=("$(dig @ns1.digitalocean.com +short A alice-workers.$NAME.dbogatov.org)")
IPS+=("$(dig @ns1.digitalocean.com +short A alice.$NAME.dbogatov.org)")
IPS=("$(dig @ns1.digitalocean.com +short A alice-workers.$NAME.dbogatov.org)")
IPS+=("$(dig @ns1.digitalocean.com +short A alice.$NAME.dbogatov.org)")
for ip in ${IPS[@]}
do
for ip in ${IPS[@]}
do
echo "Executing init script for node $ip"
echo "Executing init script for node $ip"
scp -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" node-init.sh core@$ip:/home/core
scp -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" node-init.sh core@$ip:/home/core
ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" core@$ip "sudo /home/core/node-init.sh $APIKEY"
ssh -o "StrictHostKeyChecking no" -o "UserKnownHostsFile=/dev/null" core@$ip "sudo /home/core/node-init.sh $APIKEY"
done
done
my-sleep 30
else
APITOKEN=$(cat ~/.config/digital-ocean/token)
CLUSTER_ID=$(curl -s -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" -d '{"name": "'"$NAME"'","region": "nyc1","version": "'"$VERSION"'","node_pools": [{"size": "s-1vcpu-2gb","count": 4,"name": "main-pool"}]}' "https://api.digitalocean.com/v2/kubernetes/clusters" | jq -r '.kubernetes_cluster.id')
echo "Cluster ID: $CLUSTER_ID"
STATE="init"
while [ "$STATE" != "running" ]
do
echo "Current state is $STATE"
my-sleep 10
STATE=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID" | jq -r '.kubernetes_cluster.status.state')
done
echo "State is $STATE"
curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID/kubeconfig" > kubeconfig.yaml
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
echo "Dowloaded kubefile.yaml"
EXTERNALIPS=$(kubectl $KEBEFILE get nodes -o jsonpath='{ $.items[*].status.addresses[?(@.type=="ExternalIP")].address }')
IFS=' ' read -r -a IPS <<< "$EXTERNALIPS"
curl -X DELETE --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/workers.dbogatov.org/A
for ip in "${IPS[@]}"
do
echo "Adding $ip"
curl -X POST -d "$ip" --user $EMAIL:$PASSWORD https://box.dbogatov.org/admin/dns/custom/workers.dbogatov.org/A
done
echo "Cluster provisioned!"
fi
# Let it warm up
exit
my-sleep 30
docker info > /dev/null
cd "$CWD"
......@@ -92,15 +126,15 @@ NAMESPACES=("websites" "monitoring" "ingress" "status-site" "kube-system" "gitla
for namespace in ${NAMESPACES[@]}
do
kubectl create namespace "$namespace" || true # some of them already exist
kubectl create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist
kubectl create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist
kubectl --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org
kubectl $KEBEFILE create namespace "$namespace" || true # some of them already exist
kubectl $KEBEFILE create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist
kubectl $KEBEFILE create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist
kubectl $KEBEFILE --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org || true # may already exist
done
# Save SSL certs
kubectl create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system # ???
kubectl $KEBEFILE create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist
# Deploy addons
......@@ -110,39 +144,27 @@ cd $CWD/terraform
echo "Deploying dashboard"
kubectl apply -R -f addons/dashboard/
echo "Deploying prometheus"
kubectl apply -R -f addons/prometheus/
echo "Deploying graphana"
kubectl apply -R -f addons/grafana/
echo "Deploying heapster"
kubectl apply -R -f addons/heapster/
kubectl $KEBEFILE apply -R -f addons/dashboard/
echo "Deploying NGINX Ingress"
kubectl apply -R -f addons/nginx-ingress/digital-ocean/
kubectl $KEBEFILE apply -R -f addons/nginx-ingress/digital-ocean/
cd $CWD
echo "Deploying DO volume provisioner"
./sources/do-volume-provisioner/gen-secret.sh
kubectl apply -R -f ./sources/do-volume-provisioner/
kubectl $KEBEFILE apply -R -f ./sources/do-volume-provisioner/
# For some reason must be run twice
kubectl --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml || true
kubectl --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml
kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml || true
kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml
echo "Deploying websites' settings"
kubectl create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG
kubectl create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json
kubectl $KEBEFILE create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG
kubectl $KEBEFILE create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json
echo "Generating config files"
......@@ -150,24 +172,23 @@ echo "Generating config files"
echo "Applying config files"
kubectl apply -R -f sources/nginx/
kubectl $KEBEFILE apply -R -f sources/nginx/
kubectl apply -R -f services/
kubectl $KEBEFILE apply -R -f services/
kubectl apply -R -f ./dashboard/
kubectl $KEBEFILE apply -R -f ./dashboard/
echo "Deploying status site"
kubectl apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment
# kubectl apply -f /Users/dmytro/Desktop/status-site/deployment/config.yaml
kubectl $KEBEFILE apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment
kubectl apply -R -f sources/status-site/
kubectl $KEBEFILE apply -R -f sources/status-site/
echo "Done!"
echo "Here is the dashboard login token:"
DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}"
echo $DASHBOARD_TOKEN
......
......@@ -5,20 +5,7 @@ set -e
# shopt -s globstar
source sources/data.sh
usage () {
printf "usage: ./$0 <email> <password>\n"
exit 1;
}
if ! [ $# -eq 2 ]
then
usage
fi
EMAIL=$1
PASSWORD=$2
source .secret.sh
for domain in "${!DOMAINS[@]}"
do
......
......@@ -32,7 +32,7 @@ SERVICES["visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-webs
SERVICES["eu-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/eu-visasupport-kiev-ua:latest"
SERVICES["lp-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/lp-visasupport-kiev-ua:latest"
SERVICES["zima-visasupport-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/zima-visasupport-com-ua:latest"
SERVICES["travelus-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/travelus-com-ua:latest"
# SERVICES["travelus-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/travelus-com-ua:latest"
SERVICES["visajapan-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/visajapan-com-ua:latest"
SERVICES["moon-travel-com-ua"]="registry.dbogatov.org/dbogatov/nginx-proxies/moon-travel-com-ua:latest"
......@@ -50,8 +50,7 @@ SERVICES["blog-bogatov-kiev-ua"]="registry.dbogatov.org/daddy/blog-bogatov-kiev-
declare -A DOMAINS
# TODO from DNS
AVALUE="alice-workers.arya.dbogatov.org"
AVALUE="balance.dbogatov.org"
DOMAINS["dbogatov.org"]=$AVALUE
DOMAINS["dmytro.app"]=$AVALUE
......
apiVersion: v1
data:
proxy-set-headers: "ingress/custom-headers"
add-headers: "ingress/custom-headers"
server-name-hash-bucket-size: "1024"
kind: ConfigMap
metadata:
name: nginx-configuration
namespace: ingress
labels:
app: ingress-nginx
apiVersion: v1
data:
X-Developer: "Dmytro Bogatov"
Content-Security-Policy: "upgrade-insecure-requests"
kind: ConfigMap
metadata:
name: custom-headers
namespace: ingress
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
data:
proxy-set-headers: "ingress-nginx/custom-headers"
add-headers: "ingress-nginx/custom-headers"
server-name-hash-bucket-size: "1024"
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: udp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-clusterrole
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: nginx-ingress-role
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
# Defaults to "<election-id>-<ingress-class>"
# Here: "<ingress-controller-leader>-<nginx>"
# This has to be adapted if you change either parameter
# when launching the nginx-ingress-controller.
- "ingress-controller-leader-nginx"
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: nginx-ingress-role-nisa-binding
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: nginx-ingress-role
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-clusterrole-nisa-binding
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-clusterrole
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
spec:
restartPolicy: Always
terminationGracePeriodSeconds: 60
serviceAccountName: nginx-ingress-serviceaccount
containers:
- name: nginx-ingress-controller
image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1
args:
- /nginx-ingress-controller
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
allowPrivilegeEscalation: true
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 33
runAsUser: 33
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
---
apiVersion: v1
data:
X-Developer: "Dmytro Bogatov"
Content-Security-Policy: "upgrade-insecure-requests"
kind: ConfigMap
metadata:
name: custom-headers
namespace: ingress-nginx
---
kind: Service
apiVersion: v1
metadata:
name: ingress-nginx
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
externalTrafficPolicy: Local
type: LoadBalancer
selector:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
ports:
- name: http
port: 80
targetPort: http
- name: https
port: 443
targetPort: https
---
......@@ -4,7 +4,7 @@ metadata:
name: __NAME__
namespace: websites
annotations:
kubernetes.io/ingress.class: "public"
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/ssl-redirect: "true"
nginx.ingress.kubernetes.io/from-to-www-redirect: "true"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment