From 969386b14af824b52bbac30e924a31c30194509d Mon Sep 17 00:00:00 2001 From: Dmytro Bogatov Date: Sat, 19 Oct 2019 18:16:07 -0700 Subject: [PATCH] Update to GKE. --- README.md | 21 ++---- infra/build-services.sh | 4 +- infra/migrate-spaces.sh | 7 +- infra/script.sh | 98 +++++++++++-------------- infra/setup-kubectl.sh | 9 --- infra/sources/data.sh | 10 +-- infra/test-websites.sh | 2 +- infra/upgrade-service.sh | 3 +- lets-encrypt/upload-certs-to-cluster.sh | 5 +- 9 files changed, 61 insertions(+), 98 deletions(-) delete mode 100755 infra/setup-kubectl.sh diff --git a/README.md b/README.md index 4de919f..9d3e843 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,9 @@ # Setup Manager -> When re-deploying change API key, SSH key and domain ownership! - -> Run set.dns.sh, test-websites.sh and migrate-spaces.sh - -* Follow [this](https://typhoon.psdn.io/digital-ocean/) to create a working Kubernetes cluster. -* [Create docker pull secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/). -* Use [this](https://github.com/poseidon/typhoon/blob/c3b0cdddf3310c3d6946ab018cff5229ce96623f/docs/addons/prometheus.md) to set up Prometheus-Grafana. -* Use [this](https://github.com/kubernetes/dashboard/wiki/Creating-sample-user) to get dashboard token. -* Either supply production SSL certificate (key and cert files), or [generate](https://www.digitalocean.com/community/tutorials/openssl-essentials-working-with-ssl-certificates-private-keys-and-csrs#generating-ssl-certificates) self signed files. -Make sure all domains are covered ([required](https://github.com/kubernetes/ingress-nginx/issues/616#issuecomment-359498659) by NGINX). -* Populate `infra/.secret.sh` file. -Set `OAUTH2_PROXY_CLIENT_SECRET` variable. -* Use [infra/script.sh](./infra/script.sh) to complete setup. -Example: `./infra/script.sh TOKEN ~/Desktop/certs`. +```bash +cd infra +./script.sh # e.g. ./script.sh /Users/dmytro/Desktop/certs/ our-compound-256420 +./set-dns.sh +./test-websites.sh +./migrate-spaces.sh # e.g. sep-19 oct-19 +``` diff --git a/infra/build-services.sh b/infra/build-services.sh index c6f79b5..10b1824 100755 --- a/infra/build-services.sh +++ b/infra/build-services.sh @@ -8,7 +8,7 @@ shopt -s globstar cd "${0%/*}" CWD=$(pwd) -KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml" +# KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml" source .secret.sh @@ -192,7 +192,7 @@ else mkdir -p dashboard cp ./sources/dashboard/ingress.yaml ./dashboard - DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: ) + DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: ) DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}" sed -i -e "s#__DASHBOARD_TOKEN__#$DASHBOARD_TOKEN#g" dashboard/ingress.yaml diff --git a/infra/migrate-spaces.sh b/infra/migrate-spaces.sh index c8f6aac..5b75f01 100755 --- a/infra/migrate-spaces.sh +++ b/infra/migrate-spaces.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Files in ./terraform/spaces/ must be and +# Files in ./spaces/ must be and # and must contain configs to connect to those buckets # generated with `s3cmd --configure` @@ -15,7 +15,7 @@ CWD=$(pwd) TEMP_DIR="spaces-buffer" usage () { - printf "usage: $0 \n" + printf "usage: $0 \n" exit 1; } @@ -27,7 +27,6 @@ fi FROM=$1 TO=$2 -GITLAB=$3 rm -rf $TEMP_DIR mkdir -p $TEMP_DIR @@ -36,7 +35,7 @@ cd $TEMP_DIR mkdir -p public -OBJECTS=( "gitlab-secrets.json" "$GITLAB" "public/" ) +OBJECTS=( "gitlab-secrets.json" "public/" ) for object in "${OBJECTS[@]}" do diff --git a/infra/script.sh b/infra/script.sh index 04e4bf9..f4a2168 100755 --- a/infra/script.sh +++ b/infra/script.sh @@ -8,22 +8,13 @@ shopt -s globstar cd "${0%/*}" CWD=$(pwd) -my-sleep () { - secs=$1 - while [ $secs -gt 0 ]; do - echo -ne "Waiting $secs\033[0K\r" - sleep 1 - : $((secs--)) - done -} - # Checks usage () { - printf "usage: $0 \n" + printf "usage: $0 \n" printf "where\n" printf "\t certDirPath - absolute path to directory with SSL cert (certificate.crt), key (certificate.key), auth file and appsettings for status-site\n" - printf "\t name - cluster name (e.g. sandor in sandor.dbogatov.org)\n" + printf "\t google-project-id - Google project ID\n" exit 1; } @@ -39,32 +30,38 @@ fi source .secret.sh CERTDIRPATH=$1 -NAME=$2 -KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml" +PROJECT=$2 STATUSSITECONFIG=$CERTDIRPATH/appsettings.production.yml -VERSION="1.13.10-do.1" +VERSION="1.13.10-gke.0" APIKEY=$(cat $STATUSSITECONFIG | grep "ApiKey:" | cut -d'"' -f 2) docker info > /dev/null - -APITOKEN=$(cat ~/.config/digital-ocean/token) +gcloud --version > /dev/null +# gcloud init # PROVISION -CLUSTER_ID=$(curl -s -X POST -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" -d '{"name": "'"$NAME"'","region": "nyc1","version": "'"$VERSION"'","node_pools": [{"size": "s-1vcpu-2gb","count": 3,"name": "main-pool"}]}' "https://api.digitalocean.com/v2/kubernetes/clusters" | jq -r '.kubernetes_cluster.id') -echo "Cluster ID: $CLUSTER_ID" -STATE="init" -while [ "$STATE" != "running" ] -do - echo "Current state is $STATE" - my-sleep 10 - STATE=$(curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID" | jq -r '.kubernetes_cluster.status.state') -done -echo "State is $STATE" - -curl -s -X GET -H "Content-Type: application/json" -H "Authorization: Bearer $APITOKEN" "https://api.digitalocean.com/v2/kubernetes/clusters/$CLUSTER_ID/kubeconfig" > kubeconfig.yaml -KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml" -echo "Dowloaded kubefile.yaml" +gcloud beta container --project "$PROJECT" clusters create "websites-$(date +%s)" \ + --zone "us-central1-a" \ + --no-enable-basic-auth \ + --cluster-version "$VERSION" \ + --machine-type "n1-standard-1" \ + --image-type "COS" \ + --disk-type "pd-standard" \ + --disk-size "30" \ + --metadata disable-legacy-endpoints=true \ + --scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" \ + --num-nodes "3" \ + --enable-cloud-logging \ + --enable-cloud-monitoring \ + --enable-ip-alias \ + --network "projects/$PROJECT/global/networks/default" \ + --subnetwork "projects/$PROJECT/regions/us-central1/subnetworks/default" \ + --default-max-pods-per-node "110" \ + --addons HorizontalPodAutoscaling,HttpLoadBalancing \ + --no-enable-autoupgrade \ + --enable-autorepair \ + --no-shielded-integrity-monitoring echo "Cluster provisioned!" @@ -76,39 +73,30 @@ NAMESPACES=("websites" "monitoring" "ingress" "status-site" "kube-system" "gitla for namespace in ${NAMESPACES[@]} do - kubectl $KEBEFILE create namespace "$namespace" || true # some of them already exist - kubectl $KEBEFILE create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist - kubectl $KEBEFILE create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist - kubectl $KEBEFILE --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org || true # may already exist + kubectl create namespace "$namespace" || true # some of them already exist + kubectl create --namespace="$namespace" secret tls lets-encrypt --key "$CERTDIRPATH"/certificate.key --cert "$CERTDIRPATH"/certificate.crt || true # some of them already exist + kubectl create --namespace="$namespace" secret generic basic-auth --from-file=$CERTDIRPATH/auth || true # some of them already exist + kubectl --namespace="$namespace" create secret docker-registry regsecret --docker-server=registry.dbogatov.org --docker-username=dbogatov --docker-password=$DOCKERPASS --docker-email=dmytro@dbogatov.org || true # may already exist done # Save SSL certs -kubectl $KEBEFILE create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist +kubectl create secret generic kubernetes-dashboard-certs --from-file=$CERTDIRPATH -n kube-system || true # may already exist # RESOURCES echo "Deploying dashboard" -kubectl $KEBEFILE apply -R -f ./sources/dashboard/all.yaml +kubectl apply -R -f ./sources/dashboard/all.yaml echo "Deploying NGINX Ingress" -kubectl $KEBEFILE apply -R -f ./sources/nginx/mandatory.yaml - -echo "Deploying DO volume provisioner" - -./sources/do-volume-provisioner/gen-secret.sh -kubectl $KEBEFILE apply -R -f ./sources/do-volume-provisioner/ - -# For some reason must be run twice -kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml || true -kubectl $KEBEFILE --validate=false apply -f https://raw.githubusercontent.com/digitalocean/csi-digitalocean/master/deploy/kubernetes/releases/csi-digitalocean-v0.3.1.yaml +kubectl apply -R -f ./sources/nginx/mandatory.yaml echo "Deploying websites' settings" -kubectl $KEBEFILE create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG || true # may exist -kubectl $KEBEFILE create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json || true # may exist +kubectl create secret -n status-site generic appsettings.production.yml --from-file=$STATUSSITECONFIG || true # may exist +kubectl create secret -n websites generic shevastream-appsettings --from-file=appsettings=sources/shevastream/appsettings.json || true # may exist echo "Generating config files" @@ -116,21 +104,21 @@ echo "Generating config files" echo "Applying config files" -kubectl $KEBEFILE apply -R -f services/ +kubectl apply -R -f services/ -kubectl $KEBEFILE apply -R -f ./dashboard/ingress.yaml +kubectl apply -R -f ./dashboard/ingress.yaml echo "Deploying status site" -kubectl $KEBEFILE apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment +kubectl apply -f https://git.dbogatov.org/dbogatov/status-site/-/jobs/artifacts/master/raw/deployment/config.yaml?job=release-deployment -kubectl $KEBEFILE apply -R -f sources/status-site/ +kubectl apply -R -f sources/status-site/ echo "Done!" echo "Here is the dashboard login token:" -DASHBOARD_TOKEN=$(kubectl $KEBEFILE -n kube-system describe secret $(kubectl $KEBEFILE -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: ) +DASHBOARD_TOKEN=$(kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep admin-user | awk '{print $1}') | grep token: ) DASHBOARD_TOKEN="${DASHBOARD_TOKEN:7:${#DASHBOARD_TOKEN}}" echo $DASHBOARD_TOKEN @@ -139,7 +127,7 @@ printf "\n\n" ### upgrade proxy -SERVER=$(kubectl $KEBEFILE config view -o jsonpath='{.clusters[0].cluster.server}') +SERVER=$(kubectl config view -o jsonpath='{.clusters[0].cluster.server}') cd token-proxy rm -rf ./dist @@ -154,3 +142,5 @@ cd $CWD rm -rf ./token-proxy/dist ./upgrade-service.sh token-dbogatov-org + +echo "IMPORTANT: reserve the IP adress of the Load Balancer!" diff --git a/infra/setup-kubectl.sh b/infra/setup-kubectl.sh deleted file mode 100755 index 9dd3d8e..0000000 --- a/infra/setup-kubectl.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -source .secret.sh - -K8STOKEN=$(curl -s https://$K8STOKEN@token.dbogatov.org | tr -d '[:space:]') -# KPARAMS="--insecure-skip-tls-verify=true --token=$K8STOKEN --server=https://k8sapi.dbogatov.org'" -alias kubectl='kubectl --insecure-skip-tls-verify=true --token=$K8STOKEN --server=https://k8sapi.dbogatov.org' -KUBECONFIG="" -kubectl cluster-info diff --git a/infra/sources/data.sh b/infra/sources/data.sh index 4a268a9..4959132 100644 --- a/infra/sources/data.sh +++ b/infra/sources/data.sh @@ -11,7 +11,6 @@ SERVICES["mail-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/mail SERVICES["dns-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/dns-dbogatov-org:latest" SERVICES["webcam-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/webcam-dbogatov-org:latest" SERVICES["ore-dbogatov-org"]="registry.dbogatov.org/bu/ore-benchmark/project-code/website:master" -# SERVICES["k8sapi-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/k8sapi-dbogatov-org:latest" SERVICES["cloz-dbogatov-org"]="registry.dbogatov.org/bu/ore-scheme/cloz-software-implementation:master" SERVICES["token-dbogatov-org"]="registry.dbogatov.org/dbogatov/proxy-registry:latest" SERVICES["spaces-dbogatov-org"]="registry.dbogatov.org/dbogatov/nginx-proxies/space-dbogatov-org:latest" @@ -19,10 +18,7 @@ SERVICES["budata-dbogatov-org"]="registry.dbogatov.org/bu/data-lab/website:lates SERVICES["maxflow-dbogatov-org"]="registry.dbogatov.org/bu/deduplication-project/max-flow:master" SERVICES["industry-dbogatov-org"]="registry.dbogatov.org/dbogatov/cv-website:latest" SERVICES["daria-dbogatov-org"]="registry.dbogatov.org/dorlova/orlova-app:latest" - -SERVICES["nigmatullina-org"]="registry.dbogatov.org/dbogatov/inara-cv:latest" - -# SERVICES["orlova-app"]="registry.dbogatov.org/dorlova/orlova-app:latest" +SERVICES["inara-dbogatov-org"]="registry.dbogatov.org/dbogatov/inara-cv:latest" SERVICES["darinagulley-com"]="registry.dbogatov.org/dgulley/dashawebsite:latest" @@ -34,7 +30,6 @@ SERVICES["visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-webs SERVICES["eu-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/eu-visasupport-kiev-ua:latest" SERVICES["lp-visasupport-kiev-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/lp-visasupport-kiev-ua:latest" SERVICES["zima-visasupport-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/zima-visasupport-com-ua:latest" -# SERVICES["travelus-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/travelus-com-ua:latest" SERVICES["visajapan-com-ua"]="registry.dbogatov.org/dbogatov/visasupport-websites/static/visajapan-com-ua:latest" SERVICES["moon-travel-com-ua"]="registry.dbogatov.org/dbogatov/nginx-proxies/moon-travel-com-ua:latest" @@ -59,16 +54,13 @@ DOMAINS["dmytro.app"]=$AVALUE DOMAINS["bogatov.app"]=$AVALUE DOMAINS["bogatov.dev"]=$AVALUE -# DOMAINS["orlova.app"]=$AVALUE DOMAINS["netwatch.app"]=$AVALUE DOMAINS["bogatov.kiev.ua"]=$AVALUE DOMAINS["darinagulley.com"]=$AVALUE DOMAINS["moon-travel.com.ua"]=$AVALUE -DOMAINS["nigmatullina.org"]=$AVALUE DOMAINS["photobarrat.com"]=$AVALUE DOMAINS["res-public.net"]=$AVALUE DOMAINS["shevastream.com"]=$AVALUE -# DOMAINS["travelus.com.ua"]=$AVALUE DOMAINS["veles-russia.com"]=$AVALUE DOMAINS["visajapan.com.ua"]=$AVALUE DOMAINS["visasupport.com.ua"]=$AVALUE diff --git a/infra/test-websites.sh b/infra/test-websites.sh index bab4587..26106a0 100755 --- a/infra/test-websites.sh +++ b/infra/test-websites.sh @@ -46,7 +46,7 @@ DOMAINS["darinagulley.com"]=$SUCCESS DOMAINS["moon-travel.com.ua"]=$PERMANENT_REDIRECT -DOMAINS["nigmatullina.org"]=$SUCCESS +# DOMAINS["nigmatullina.org"]=$SUCCESS # DOMAINS["photobarrat.com"]=$SUCCESS diff --git a/infra/upgrade-service.sh b/infra/upgrade-service.sh index 6733eb7..b3a10e0 100755 --- a/infra/upgrade-service.sh +++ b/infra/upgrade-service.sh @@ -10,7 +10,6 @@ CWD=$(pwd) usage() { echo "Usage: $0 [-s ]" 1>&2; exit 1; } -KEBEFILE="--kubeconfig=${CWD}/kubeconfig.yaml" SERVICE="" while getopts "s:" o; do @@ -26,6 +25,6 @@ done shift $((OPTIND-1)) source ./build-services.sh $SERVICE -kubectl $KEBEFILE apply -R -f ./services/$SERVICE +kubectl apply -R -f ./services/$SERVICE echo "Done!" diff --git a/lets-encrypt/upload-certs-to-cluster.sh b/lets-encrypt/upload-certs-to-cluster.sh index 8bead25..ab3fb85 100755 --- a/lets-encrypt/upload-certs-to-cluster.sh +++ b/lets-encrypt/upload-certs-to-cluster.sh @@ -21,15 +21,14 @@ then usage fi -KEBEFILE="--kubeconfig=${CWD}/../infra/kubeconfig.yaml" CERTDIRPATH=$1 NAMESPACES=("websites" "monitoring" "ingress" "status-site" "kube-system" "gitlab" "review") for namespace in ${NAMESPACES[@]} do - kubectl $KEBEFILE delete --namespace=$namespace secret lets-encrypt || true - kubectl $KEBEFILE create --namespace=$namespace secret tls lets-encrypt --key $CERTDIRPATH/certificate.key --cert $CERTDIRPATH/certificate.crt || true + kubectl delete --namespace=$namespace secret lets-encrypt || true + kubectl create --namespace=$namespace secret tls lets-encrypt --key $CERTDIRPATH/certificate.key --cert $CERTDIRPATH/certificate.crt || true done echo "Done." -- GitLab