Commit 969386b1 authored by Dmytro Bogatov's avatar Dmytro Bogatov 💕

Update to GKE.

parent 76721f5d
Pipeline #5192 passed with stage
in 3 seconds
# Setup Manager
> When re-deploying change API key, SSH key and domain ownership!
> Run set.dns.sh, test-websites.sh and migrate-spaces.sh
* Follow [this](https://typhoon.psdn.io/digital-ocean/) to create a working Kubernetes cluster.
* [Create docker pull secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
* Use [this](https://github.com/poseidon/typhoon/blob/c3b0cdddf3310c3d6946ab018cff5229ce96623f/docs/addons/prometheus.md) to set up Prometheus-Grafana.
* Use [this](https://github.com/kubernetes/dashboard/wiki/Creating-sample-user) to get dashboard token.
* Either supply production SSL certificate (key and cert files), or [generate](https://www.digitalocean.com/community/tutorials/openssl-essentials-working-with-ssl-certificates-private-keys-and-csrs#generating-ssl-certificates) self signed files.
Make sure all domains are covered ([required](https://github.com/kubernetes/ingress-nginx/issues/616#issuecomment-359498659) by NGINX).
* Populate `infra/.secret.sh` file.
Set `OAUTH2_PROXY_CLIENT_SECRET` variable.
* Use [infra/script.sh](./infra/script.sh) to complete setup.
Example: `./infra/script.sh TOKEN ~/Desktop/certs`.
```bash
cd infra
./script.sh # e.g. ./script.sh /Users/dmytro/Desktop/certs/ our-compound-256420
./set-dns.sh
./test-websites.sh
./migrate-spaces.sh # e.g. sep-19 oct-19
```
......@@ -8,7 +8,7 @@ shopt -s globstar
cd "${0%/*}"
CWD=$(pwd)
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
# KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
source .secret.sh
......@@ -192,7 +192,7 @@ else
mkdir -p dashboard
cp ./sources/dashboard/ingress.yaml ./dashboard
DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}"
sed -i -e "s#__DASHBOARD_TOKEN__#$DASHBOARD_TOKEN#g" dashboard/ingress.yaml
......
#!/usr/bin/env bash
# Files in ./terraform/spaces/ must be <from> and <to>
# Files in ./spaces/ must be <from> and <to>
# and must contain configs to connect to those buckets
# generated with `s3cmd --configure`
......@@ -15,7 +15,7 @@ CWD=$(pwd)
TEMP_DIR="spaces-buffer"
usage () {
printf "usage: $0 <from> <to> <last-gitlab-backup>\n"
printf "usage: $0 <from> <to>\n"
exit 1;
}
......@@ -27,7 +27,6 @@ fi
FROM=$1
TO=$2
GITLAB=$3
rm -rf $TEMP_DIR
mkdir -p $TEMP_DIR
......@@ -36,7 +35,7 @@ cd $TEMP_DIR
mkdir -p public
OBJECTS=( "gitlab-secrets.json" "$GITLAB" "public/" )
OBJECTS=( "gitlab-secrets.json" "public/" )
for object in "${OBJECTS[@]}"
do
......
......@@ -8,22 +8,13 @@ shopt -s globstar
cd "${0%/*}"
CWD=$(pwd)
my-sleep () {
secs=$1
while [ $secs -gt 0 ]; do
echo -ne "Waiting $secs\033[0K\r"
sleep 1
: $((secs--))
done
}
# Checks
usage () {
printf "usage: $0 <certDirPath> <name>\n"
printf "usage: $0 <certDirPath> <google-project-id>\n"
printf "where\n"
printf "\t certDirPath - absolute path to directory with SSL cert (certificate.crt), key (certificate.key), auth file and appsettings for status-site\n"
printf "\t name - cluster name (e.g. sandor in sandor.dbogatov.org)\n"
printf "\t google-project-id - Google project ID\n"
exit 1;
}
......@@ -39,32 +30,38 @@ fi
source .secret.sh
CERTDIRPATH=$1
NAME=$2
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
PROJECT=$2
STATUSSITECONFIG=$CERTDIRPATH/appsettings.production.yml
VERSION="1.13.10-do.1"
VERSION="1.13.10-gke.0"
APIKEY=$(cat $STATUSSITECONFIG | grep "ApiKey:" | cut -d'"' -f 2)
docker info > /dev/null
APITOKEN=$(cat ~/.config/digital-ocean/token)
gcloud --version > /dev/null
# gcloud init
# PROVISION
CLUSTER_ID=$(curl -s -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" -d '{"name": "'"$NAME"'","region": "nyc1","version": "'"$VERSION"'","node_pools": [{"size": "s-1vcpu-2gb","count": 3,"name": "main-pool"}]}' "https://api.digitalocean.com/v2/kubernetes/clusters" | jq -r '.kubernetes_cluster.id')
echo "Cluster ID: $CLUSTER_ID"
STATE="init"
while [ "$STATE" != "running" ]
do
echo "Current state is $STATE"
my-sleep 10
STATE=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID" | jq -r '.kubernetes_cluster.status.state')
done
echo "State is $STATE"
curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID/kubeconfig" > kubeconfig.yaml
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
echo "Dowloaded kubefile.yaml"
gcloud beta container --project "$PROJECT" clusters create "websites-$(date +%s)" \
--zone "us-central1-a" \
--no-enable-basic-auth \
--cluster-version "$VERSION" \
--machine-type "n1-standard-1" \
--image-type "COS" \
--disk-type "pd-standard" \
--disk-size "30" \
--metadata disable-legacy-endpoints=true \
--scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" \
--num-nodes "3" \
--enable-cloud-logging \
--enable-cloud-monitoring \
--enable-ip-alias \
--network "projects/$PROJECT/global/networks/default" \
--subnetwork "projects/$PROJECT/regions/us-central1/subnetworks/default" \
--default-max-pods-per-node "110" \
--addons HorizontalPodAutoscaling,HttpLoadBalancing \
--no-enable-autoupgrade \
--enable-autorepair \
--no-shielded-integrity-monitoring
echo "Cluster provisioned!"
......@@ -76,39 +73,30 @@ NAMESPACES=("websites" "monitoring" "ingress" "status-site" "kube-system" "gitla
for namespace in ${NAMESPACES[@]}
do
kubectl $KEBEFILE create namespace "$namespace" || true # some of them already exist
kubectl $KEBEFILE create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist
kubectl $KEBEFILE create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist
kubectl $KEBEFILE --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org || true # may already exist
kubectl create namespace "$namespace" || true # some of them already exist
kubectl create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist
kubectl create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist
kubectl --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org || true # may already exist
done
# Save SSL certs
kubectl $KEBEFILE create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist
kubectl create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist
# RESOURCES
echo "Deploying dashboard"
kubectl $KEBEFILE apply -R -f ./sources/dashboard/all.yaml
kubectl apply -R -f ./sources/dashboard/all.yaml
echo "Deploying NGINX Ingress"
kubectl $KEBEFILE apply -R -f ./sources/nginx/mandatory.yaml
echo "Deploying DO volume provisioner"
./sources/do-volume-provisioner/gen-secret.sh
kubectl $KEBEFILE apply -R -f ./sources/do-volume-provisioner/
# For some reason must be run twice
kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml || true
kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml
kubectl apply -R -f ./sources/nginx/mandatory.yaml
echo "Deploying websites' settings"
kubectl $KEBEFILE create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG || true # may exist
kubectl $KEBEFILE create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json || true # may exist
kubectl create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG || true # may exist
kubectl create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json || true # may exist
echo "Generating config files"
......@@ -116,21 +104,21 @@ echo "Generating config files"
echo "Applying config files"
kubectl $KEBEFILE apply -R -f services/
kubectl apply -R -f services/
kubectl $KEBEFILE apply -R -f ./dashboard/ingress.yaml
kubectl apply -R -f ./dashboard/ingress.yaml
echo "Deploying status site"
kubectl $KEBEFILE apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment
kubectl apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment
kubectl $KEBEFILE apply -R -f sources/status-site/
kubectl apply -R -f sources/status-site/
echo "Done!"
echo "Here is the dashboard login token:"
DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: )
DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}"
echo $DASHBOARD_TOKEN
......@@ -139,7 +127,7 @@ printf "\n\n"
### upgrade proxy
SERVER=$(kubectl $KEBEFILE config view -o jsonpath='{.clusters[0].cluster.server}')
SERVER=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}')
cd token-proxy
rm -rf ./dist
......@@ -154,3 +142,5 @@ cd $CWD
rm -rf ./token-proxy/dist
./upgrade-service.sh token-dbogatov-org
echo "IMPORTANT: reserve the IP adress of the Load Balancer!"
#!/usr/bin/env bash
source .secret.sh
K8STOKEN=$(curl -s https://$K8STOKEN@token.dbogatov.org | tr -d '[:space:]')
# KPARAMS="--insecure-skip-tls-verify=true --token=$K8STOKEN --server=https://k8sapi.dbogatov.org'"
alias kubectl='kubectl --insecure-skip-tls-verify=true --token=$K8STOKEN --server=https://k8sapi.dbogatov.org'
KUBECONFIG=""
kubectl cluster-info
......@@ -11,7 +11,6 @@ SERVICES["mail-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/mail
SERVICES["dns-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/dns-dbogatov-org:latest"
SERVICES["webcam-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/webcam-dbogatov-org:latest"
SERVICES["ore-dbogatov-org"]="registry.dbogatov.org/bu/ore-benchmark/project-code/website:master"
# SERVICES["k8sapi-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/k8sapi-dbogatov-org:latest"
SERVICES["cloz-dbogatov-org"]="registry.dbogatov.org/bu/ore-scheme/cloz-software-implementation:master"
SERVICES["token-dbogatov-org"]="registry.dbogatov.org/dbogatov/proxy-registry:latest"
SERVICES["spaces-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/space-dbogatov-org:latest"
......@@ -19,10 +18,7 @@ SERVICES["budata-dbogatov-org"]="registry.dbogatov.org/bu/data-lab/website:lates
SERVICES["maxflow-dbogatov-org"]="registry.dbogatov.org/bu/deduplication-project/max-flow:master"
SERVICES["industry-dbogatov-org"]="registry.dbogatov.org/dbogatov/cv-website:latest"
SERVICES["daria-dbogatov-org"]="registry.dbogatov.org/dorlova/orlova-app:latest"
SERVICES["nigmatullina-org"]="registry.dbogatov.org/dbogatov/inara-cv:latest"
# SERVICES["orlova-app"]="registry.dbogatov.org/dorlova/orlova-app:latest"
SERVICES["inara-dbogatov-org"]="registry.dbogatov.org/dbogatov/inara-cv:latest"
SERVICES["darinagulley-com"]="registry.dbogatov.org/dgulley/dashawebsite:latest"
......@@ -34,7 +30,6 @@ SERVICES["visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-webs
SERVICES["eu-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/eu-visasupport-kiev-ua:latest"
SERVICES["lp-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/lp-visasupport-kiev-ua:latest"
SERVICES["zima-visasupport-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/zima-visasupport-com-ua:latest"
# SERVICES["travelus-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/travelus-com-ua:latest"
SERVICES["visajapan-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/visajapan-com-ua:latest"
SERVICES["moon-travel-com-ua"]="registry.dbogatov.org/dbogatov/nginx-proxies/moon-travel-com-ua:latest"
......@@ -59,16 +54,13 @@ DOMAINS["dmytro.app"]=$AVALUE
DOMAINS["bogatov.app"]=$AVALUE
DOMAINS["bogatov.dev"]=$AVALUE
# DOMAINS["orlova.app"]=$AVALUE
DOMAINS["netwatch.app"]=$AVALUE
DOMAINS["bogatov.kiev.ua"]=$AVALUE
DOMAINS["darinagulley.com"]=$AVALUE
DOMAINS["moon-travel.com.ua"]=$AVALUE
DOMAINS["nigmatullina.org"]=$AVALUE
DOMAINS["photobarrat.com"]=$AVALUE
DOMAINS["res-public.net"]=$AVALUE
DOMAINS["shevastream.com"]=$AVALUE
# DOMAINS["travelus.com.ua"]=$AVALUE
DOMAINS["veles-russia.com"]=$AVALUE
DOMAINS["visajapan.com.ua"]=$AVALUE
DOMAINS["visasupport.com.ua"]=$AVALUE
......
......@@ -46,7 +46,7 @@ DOMAINS["darinagulley.com"]=$SUCCESS
DOMAINS["moon-travel.com.ua"]=$PERMANENT_REDIRECT
DOMAINS["nigmatullina.org"]=$SUCCESS
# DOMAINS["nigmatullina.org"]=$SUCCESS
# DOMAINS["photobarrat.com"]=$SUCCESS
......
......@@ -10,7 +10,6 @@ CWD=$(pwd)
usage() { echo "Usage: $0 [-s <string>]" 1>&2; exit 1; }
KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml"
SERVICE=""
while getopts "s:" o; do
......@@ -26,6 +25,6 @@ done
shift $((OPTIND-1))
source ./build-services.sh $SERVICE
kubectl $KEBEFILE apply -R -f ./services/$SERVICE
kubectl apply -R -f ./services/$SERVICE
echo "Done!"
......@@ -21,15 +21,14 @@ then
usage
fi
KEBEFILE="--kubeconfig=${CWD}/../infra/kubeconfig.yaml"
CERTDIRPATH=$1
NAMESPACES=("websites" "monitoring" "ingress" "status-site" "kube-system" "gitlab" "review")
for namespace in ${NAMESPACES[@]}
do
kubectl $KEBEFILE delete --namespace=$namespace secret lets-encrypt || true
kubectl $KEBEFILE create --namespace=$namespace secret tls lets-encrypt --key $CERTDIRPATH/certificate.key --cert $CERTDIRPATH/certificate.crt || true
kubectl delete --namespace=$namespace secret lets-encrypt || true
kubectl create --namespace=$namespace secret tls lets-encrypt --key $CERTDIRPATH/certificate.key --cert $CERTDIRPATH/certificate.crt || true
done
echo "Done."
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment