Update to GKE.

969386b1 · Dmytro Bogatov · 76721f5d · 969386b1 · 969386b1 · 969386b1
Commit 969386b1 authored Oct 19, 2019 by Dmytro Bogatov 💕
9 changed files
--- a/README.md
+++ b/README.md
 # Setup Manager

-> When re-deploying change API key, SSH key and domain ownership!
-
-> Run set.dns.sh, test-websites.sh and migrate-spaces.sh
-
-* Follow [this](https://typhoon.psdn.io/digital-ocean/) to create a working Kubernetes cluster.
-* [Create docker pull secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
-* Use [this](https://github.com/poseidon/typhoon/blob/c3b0cdddf3310c3d6946ab018cff5229ce96623f/docs/addons/prometheus.md) to set up Prometheus-Grafana.
-* Use [this](https://github.com/kubernetes/dashboard/wiki/Creating-sample-user) to get dashboard token.
-* Either supply production SSL certificate (key and cert files), or [generate](https://www.digitalocean.com/community/tutorials/openssl-essentials-working-with-ssl-certificates-private-keys-and-csrs#generating-ssl-certificates) self signed files.
-Make sure all domains are covered ([required](https://github.com/kubernetes/ingress-nginx/issues/616#issuecomment-359498659) by NGINX).
-* Populate `infra/.secret.sh` file.
-Set `OAUTH2_PROXY_CLIENT_SECRET` variable.
-* Use [infra/script.sh](./infra/script.sh) to complete setup.
-Example: `./infra/script.sh TOKEN ~/Desktop/certs`.
+```bash
+cd infra
+./script.sh # e.g. ./script.sh /Users/dmytro/Desktop/certs/ our-compound-256420
+./set-dns.sh
+./test-websites.sh
+./migrate-spaces.sh # e.g. sep-19 oct-19
+```
--- a/infra/build-services.sh
+++ b/infra/build-services.sh
@@ -8,7 +8,7 @@ shopt -s globstar
 cd "${0%/*}"
 CWD=$(pwd)

-KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
+# KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"

 source .secret.sh

@@ -192,7 +192,7 @@ else
 		mkdir -p dashboard
 		cp ./sources/dashboard/ingress.yaml ./dashboard

-		DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
+		DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
 		DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}"

 		sed -i -e "s#__DASHBOARD_TOKEN__#$DASHBOARD_TOKEN#g" dashboard/ingress.yaml

--- a/infra/migrate-spaces.sh
+++ b/infra/migrate-spaces.sh
 #!/usr/bin/env bash

-# Files in ./terraform/spaces/ must be <from> and <to>
+# Files in ./spaces/ must be <from> and <to>
 # and must contain configs to connect to those buckets
 # generated with `s3cmd --configure`

@@ -15,7 +15,7 @@ CWD=$(pwd)
 TEMP_DIR="spaces-buffer"

 usage () {
-	printf "usage: $0 <from> <to> <last-gitlab-backup>\n"
+	printf "usage: $0 <from> <to>\n"

 	exit 1;
 }
@@ -27,7 +27,6 @@ fi

 FROM=$1
 TO=$2
-GITLAB=$3

 rm -rf $TEMP_DIR
 mkdir -p $TEMP_DIR
@@ -36,7 +35,7 @@ cd $TEMP_DIR

 mkdir -p public

-OBJECTS=( "gitlab-secrets.json" "$GITLAB" "public/" )
+OBJECTS=( "gitlab-secrets.json" "public/" )

 for object in "${OBJECTS[@]}"
 do

--- a/infra/script.sh
+++ b/infra/script.sh
@@ -8,22 +8,13 @@ shopt -s globstar
 cd "${0%/*}"
 CWD=$(pwd)

-my-sleep () {
-	secs=$1
-	while [ $secs -gt 0 ]; do
-		echo -ne "Waiting $secs\033[0K\r"
-		sleep 1
-		: $((secs--))
-	done
-}
-
 # Checks

 usage () {
-	printf "usage: $0 <certDirPath> <name>\n"
+	printf "usage: $0 <certDirPath> <google-project-id>\n"
 	printf "where\n"
 	printf "\t certDirPath - absolute path to directory with SSL cert (certificate.crt), key (certificate.key), auth file and appsettings for status-site\n"
-	printf "\t name - cluster name (e.g. sandor in sandor.dbogatov.org)\n"
+	printf "\t google-project-id - Google project ID\n"

 	exit 1;
 }
@@ -39,32 +30,38 @@ fi
 source .secret.sh

 CERTDIRPATH=$1
-NAME=$2
-KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
+PROJECT=$2
 STATUSSITECONFIG=$CERTDIRPATH/appsettings.production.yml
-VERSION="1.13.10-do.1"
+VERSION="1.13.10-gke.0"
 APIKEY=$(cat $STATUSSITECONFIG | grep "ApiKey:" | cut -d'"' -f 2)

 docker info > /dev/null
-
-APITOKEN=$(cat ~/.config/digital-ocean/token)
+gcloud --version > /dev/null
+# gcloud init

 # PROVISION

-CLUSTER_ID=$(curl -s -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" -d '{"name": "'"$NAME"'","region": "nyc1","version": "'"$VERSION"'","node_pools": [{"size": "s-1vcpu-2gb","count": 3,"name": "main-pool"}]}' "https://api.digitalocean.com/v2/kubernetes/clusters" | jq -r '.kubernetes_cluster.id')
-echo "Cluster ID: $CLUSTER_ID"
-STATE="init"
-while [ "$STATE" != "running" ]
-do
-	echo "Current state is $STATE"
-	my-sleep 10
-	STATE=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID" | jq -r '.kubernetes_cluster.status.state')
-done
-echo "State is $STATE"
-
-curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID/kubeconfig" > kubeconfig.yaml
-KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
-echo "Dowloaded kubefile.yaml"
+gcloud beta container --project "$PROJECT" clusters create "websites-$(date +%s)" \
+	--zone "us-central1-a" \
+	--no-enable-basic-auth \
+	--cluster-version "$VERSION" \
+	--machine-type "n1-standard-1" \
+	--image-type "COS" \
+	--disk-type "pd-standard" \
+	--disk-size "30" \
+	--metadata disable-legacy-endpoints=true \
+	--scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" \
+	--num-nodes "3" \
+	--enable-cloud-logging \
+	--enable-cloud-monitoring \
+	--enable-ip-alias \
+	--network "projects/$PROJECT/global/networks/default" \
+	--subnetwork "projects/$PROJECT/regions/us-central1/subnetworks/default" \
+	--default-max-pods-per-node "110" \
+	--addons HorizontalPodAutoscaling,HttpLoadBalancing \
+	--no-enable-autoupgrade \
+	--enable-autorepair \
+	--no-shielded-integrity-monitoring

 echo "Cluster provisioned!"

@@ -76,39 +73,30 @@ NAMESPACES=("websites" "monitoring" "ingress" "status-site" "kube-system" "gitla

 for namespace in ${NAMESPACES[@]}
 do
-	kubectl $KEBEFILE create namespace "$namespace" || true # some of them already exist
-	kubectl $KEBEFILE create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist
-	kubectl $KEBEFILE create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist
-	kubectl $KEBEFILE --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org || true # may already exist
+	kubectl create namespace "$namespace" || true # some of them already exist
+	kubectl create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist
+	kubectl create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist
+	kubectl --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org || true # may already exist
 done

 # Save SSL certs

-kubectl $KEBEFILE create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist
+kubectl create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist

 # RESOURCES

 echo "Deploying dashboard"

-kubectl $KEBEFILE apply -R -f ./sources/dashboard/all.yaml
+kubectl apply -R -f ./sources/dashboard/all.yaml

 echo "Deploying NGINX Ingress"

-kubectl $KEBEFILE apply -R -f ./sources/nginx/mandatory.yaml
-
-echo "Deploying DO volume provisioner"
-
-./sources/do-volume-provisioner/gen-secret.sh
-kubectl $KEBEFILE apply -R -f ./sources/do-volume-provisioner/
-
-# For some reason must be run twice
-kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml || true
-kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml
+kubectl apply -R -f ./sources/nginx/mandatory.yaml

 echo "Deploying websites' settings"

-kubectl $KEBEFILE create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG || true # may exist
-kubectl $KEBEFILE create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json || true # may exist
+kubectl create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG || true # may exist
+kubectl create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json || true # may exist

 echo "Generating config files"

@@ -116,21 +104,21 @@ echo "Generating config files"

 echo "Applying config files"

-kubectl $KEBEFILE apply -R -f services/
+kubectl apply -R -f services/

-kubectl $KEBEFILE apply -R -f ./dashboard/ingress.yaml
+kubectl apply -R -f ./dashboard/ingress.yaml

 echo "Deploying status site"

-kubectl $KEBEFILE apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment
+kubectl apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment

-kubectl $KEBEFILE apply -R -f sources/status-site/
+kubectl apply -R -f sources/status-site/

 echo "Done!"

 echo "Here is the dashboard login token:"

-DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
+DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
 DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}"

 echo $DASHBOARD_TOKEN
@@ -139,7 +127,7 @@ printf "\n\n"

 ### upgrade proxy

-SERVER=$(kubectl $KEBEFILE config view -o jsonpath='{.clusters[0].cluster.server}')
+SERVER=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}')

 cd token-proxy
 rm -rf ./dist
@@ -154,3 +142,5 @@ cd $CWD
 rm -rf ./token-proxy/dist

 ./upgrade-service.sh token-dbogatov-org
+
+echo "IMPORTANT: reserve the IP adress of the Load Balancer!"
--- a/infra/setup-kubectl.sh
+++ b/infra/setup-kubectl.sh
-#!/usr/bin/env bash
-
-source .secret.sh
-
-K8STOKEN=$(curl -s https://$K8STOKEN@token.dbogatov.org | tr -d '[:space:]')
-# KPARAMS="--insecure-skip-tls-verify=true --token=$K8STOKEN --server=https://k8sapi.dbogatov.org'"
-alias kubectl='kubectl --insecure-skip-tls-verify=true --token=$K8STOKEN --server=https://k8sapi.dbogatov.org'
-KUBECONFIG=""
-kubectl cluster-info
--- a/infra/sources/data.sh
+++ b/infra/sources/data.sh
@@ -11,7 +11,6 @@ SERVICES["mail-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/mail
 SERVICES["dns-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/dns-dbogatov-org:latest"
 SERVICES["webcam-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/webcam-dbogatov-org:latest"
 SERVICES["ore-dbogatov-org"]="registry.dbogatov.org/bu/ore-benchmark/project-code/website:master"
-# SERVICES["k8sapi-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/k8sapi-dbogatov-org:latest"
 SERVICES["cloz-dbogatov-org"]="registry.dbogatov.org/bu/ore-scheme/cloz-software-implementation:master"
 SERVICES["token-dbogatov-org"]="registry.dbogatov.org/dbogatov/proxy-registry:latest"
 SERVICES["spaces-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/space-dbogatov-org:latest"
@@ -19,10 +18,7 @@ SERVICES["budata-dbogatov-org"]="registry.dbogatov.org/bu/data-lab/website:lates
 SERVICES["maxflow-dbogatov-org"]="registry.dbogatov.org/bu/deduplication-project/max-flow:master"
 SERVICES["industry-dbogatov-org"]="registry.dbogatov.org/dbogatov/cv-website:latest"
 SERVICES["daria-dbogatov-org"]="registry.dbogatov.org/dorlova/orlova-app:latest"
-
-SERVICES["nigmatullina-org"]="registry.dbogatov.org/dbogatov/inara-cv:latest"
-
-# SERVICES["orlova-app"]="registry.dbogatov.org/dorlova/orlova-app:latest"
+SERVICES["inara-dbogatov-org"]="registry.dbogatov.org/dbogatov/inara-cv:latest"

 SERVICES["darinagulley-com"]="registry.dbogatov.org/dgulley/dashawebsite:latest"

@@ -34,7 +30,6 @@ SERVICES["visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-webs
 SERVICES["eu-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/eu-visasupport-kiev-ua:latest"
 SERVICES["lp-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/lp-visasupport-kiev-ua:latest"
 SERVICES["zima-visasupport-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/zima-visasupport-com-ua:latest"
-# SERVICES["travelus-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/travelus-com-ua:latest"
 SERVICES["visajapan-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/visajapan-com-ua:latest"

 SERVICES["moon-travel-com-ua"]="registry.dbogatov.org/dbogatov/nginx-proxies/moon-travel-com-ua:latest"
@@ -59,16 +54,13 @@ DOMAINS["dmytro.app"]=$AVALUE
 DOMAINS["bogatov.app"]=$AVALUE
 DOMAINS["bogatov.dev"]=$AVALUE

-# DOMAINS["orlova.app"]=$AVALUE
 DOMAINS["netwatch.app"]=$AVALUE
 DOMAINS["bogatov.kiev.ua"]=$AVALUE
 DOMAINS["darinagulley.com"]=$AVALUE
 DOMAINS["moon-travel.com.ua"]=$AVALUE
-DOMAINS["nigmatullina.org"]=$AVALUE
 DOMAINS["photobarrat.com"]=$AVALUE
 DOMAINS["res-public.net"]=$AVALUE
 DOMAINS["shevastream.com"]=$AVALUE
-# DOMAINS["travelus.com.ua"]=$AVALUE
 DOMAINS["veles-russia.com"]=$AVALUE
 DOMAINS["visajapan.com.ua"]=$AVALUE
 DOMAINS["visasupport.com.ua"]=$AVALUE

--- a/infra/test-websites.sh
+++ b/infra/test-websites.sh
@@ -46,7 +46,7 @@ DOMAINS["darinagulley.com"]=$SUCCESS

 DOMAINS["moon-travel.com.ua"]=$PERMANENT_REDIRECT

-DOMAINS["nigmatullina.org"]=$SUCCESS
+# DOMAINS["nigmatullina.org"]=$SUCCESS

 # DOMAINS["photobarrat.com"]=$SUCCESS


--- a/infra/upgrade-service.sh
+++ b/infra/upgrade-service.sh
@@ -10,7 +10,6 @@ CWD=$(pwd)

 usage() { echo "Usage: $0 [-s <string>]" 1>&2; exit 1; }

-KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
 SERVICE=""

 while getopts "s:" o; do
@@ -26,6 +25,6 @@ done
 shift $((OPTIND-1))

 source ./build-services.sh $SERVICE
-kubectl $KEBEFILE apply -R -f ./services/$SERVICE
+kubectl apply -R -f ./services/$SERVICE

 echo "Done!"
--- a/lets-encrypt/upload-certs-to-cluster.sh
+++ b/lets-encrypt/upload-certs-to-cluster.sh
@@ -21,15 +21,14 @@ then
 	usage
 fi

-KEBEFILE="--kubeconfig=${CWD}/../infra/kubeconfig.yaml"
 CERTDIRPATH=$1

 NAMESPACES=("websites" "monitoring" "ingress" "status-site" "kube-system" "gitlab" "review")

 for namespace in ${NAMESPACES[@]}
 do
-	kubectl $KEBEFILE delete --namespace=$namespace secret lets-encrypt || true
-	kubectl $KEBEFILE create --namespace=$namespace secret tls lets-encrypt --key $CERTDIRPATH/certificate.key --cert $CERTDIRPATH/certificate.crt || true
+	kubectl delete --namespace=$namespace secret lets-encrypt || true
+	kubectl create --namespace=$namespace secret tls lets-encrypt --key $CERTDIRPATH/certificate.key --cert $CERTDIRPATH/certificate.crt || true
 done

 echo "Done."