Merge branch '16-separate-ping-container' into 'master'

Resolve "Separate ping container"

Closes #16

See merge request !7

.gitignore

@@ -51,3 +51,5 @@ articles/site/
 
 # Documentation
 documentation/out/
+
+debug

.gitlab-ci.yml

@@ -20,6 +20,20 @@ build-docs:
   tags:
   - docker
 
+build-ping:
+  image: golang:alpine
+  stage: build
+  script:
+  - apk update
+  - apk add bash
+  - ./build.sh -f build-ping-server
+  artifacts:
+    expire_in: 90 min
+    paths:
+    - ping/bin/*
+  tags:
+  - docker
+
 build-app:
   image: dbogatov/docker-containers:dotnet-core-latest
   stage: build
@@ -101,6 +115,7 @@ release-all:
   dependencies:
   - build-app
   - build-docs
+  - build-ping
   script:
   - ./build.sh -f build-docker-images
   - docker login -u $DOCKER_USER -p $DOCKER_PASS

.vscode/launch.json

 {
 	"version": "0.2.0",
 	"configurations": [
-		
+		{
+			"name": "go (console)",
+			"type": "go",
+			"request": "launch",
+			"mode": "debug",
+			"program": "${file}"
+		},
 		{
 			"name": "web (console)",
 			"type": "coreclr",

.vscode/settings.json

 // Place your settings in this file to overwrite default and user settings.
 {
-	"files.autoSave": "onFocusChange"
 }

articles/docs/configuration.md

@@ -107,6 +107,7 @@ Configuration spec:
 		NotificationService: 								# Settings for notification service of the app
 			Enabled: true 									# Whether to use the service
 			Interval: 20									# How many seconds to wait between re-runs of the service
+			TimeZone: "America/New_York"					# Preferred time zone for displaying dates and times in notifications
 			Frequencies: 									# Number of seconds to wait before sending out notifications of given severity
 				Low: 86400 									# Example: notifications of low severity will be sent no more than once in 86400 seconds
 				Medium: 360 								# Example: notifications of medium severity will be sent no more than once in 360 seconds

build.sh

@@ -161,6 +161,14 @@ move-static-files () {
 	chmod -x documentation/out/deploy.sh
 }
 
+build-ping-server () {
+	cd $CWD/ping
+
+	echo "Building ping server..."
+
+	go build -o bin/ping main/main.go 
+}
+
 build-dev-client () {
 
 	cd $CWD
@@ -207,6 +215,9 @@ build-docker-images () {
 	echo "Building nginx-$DOTNET_TAG"
 	docker build -f nginx/Dockerfile -t dbogatov/status-site:nginx-$DOTNET_TAG nginx/
 
+	echo "Building ping-$DOTNET_TAG"
+	docker build -f ping/Dockerfile -t dbogatov/status-site:ping-$DOTNET_TAG ping/
+
 	echo "Done!"
 }
 
@@ -228,6 +239,9 @@ push-docker-images () {
 	echo "Pushing nginx-$DOTNET_TAG"
 	docker push dbogatov/status-site:nginx-$DOTNET_TAG
 
+	echo "Pushing ping-$DOTNET_TAG"
+	docker push dbogatov/status-site:ping-$DOTNET_TAG
+
 	echo "Done!"
 }
 

client/ts/modules/metric-page/ping.ts

@@ -36,7 +36,7 @@ export class PingMetricPage extends MetricPage<Metric<PingDataPoint>> {
 			.reverse()
 			.forEach(
 			(value, index, array) => {
-				if (value.httpStatusCode == 200) {
+				if (value.success) {
 					data.push([value.timestamp.getTime(), value.responseTime]);
 				} else {
 					errors.push([value.timestamp.getTime(), this.max]);
@@ -157,7 +157,7 @@ export class PingMetricPage extends MetricPage<Metric<PingDataPoint>> {
 				<tr>
 					<th>Timestamp</th>
 					<th>Latency</th>
-					<th>Response Code</th>
+					<th>Result</th>
 				</tr>
 			`;
 
@@ -173,7 +173,7 @@ export class PingMetricPage extends MetricPage<Metric<PingDataPoint>> {
 						<tr>
 							<td>${dp.timestamp}</td>
 							<td>${dp.responseTime} ms</td>
-							<td>${dp.httpStatusCode}</td>
+							<td>${dp.success ? "Success" : dp.message}</td>
 						</tr>
 					`
 					)

client/ts/modules/metrics/ping.ts

@@ -8,7 +8,8 @@ import "../../vendor/jquery.flot.tooltip.js";
 type JsonPingDataPoint = {
 	Timestamp: string;
 	ResponseTime: number;
-	HttpStatusCode: number;
+	Success: boolean;
+	Message: string;
 }
 
 /**
@@ -21,14 +22,16 @@ type JsonPingDataPoint = {
 export class PingDataPoint extends DataPoint {
 
 	public responseTime: number;
-	public httpStatusCode: number;
+	public success: boolean;
+	public message: string;
 
 	constructor(json: JsonPingDataPoint) {
 		super();
 
 		this.timestamp = Utility.toDate(json.Timestamp);
 		this.responseTime = json.ResponseTime;
-		this.httpStatusCode = json.HttpStatusCode;
+		this.success = json.Success;
+		this.message = json.Message;
 	}
 }
 
@@ -56,7 +59,7 @@ export class PingMetric extends Metric<PingDataPoint> {
 			.sortByProperty(dp => dp.timestamp.getTime())
 			.forEach(
 			(value, index, array) => {
-				if (value.httpStatusCode == 200) {
+				if (value.success) {
 					data.push([index, value.responseTime]);
 				} else {
 					errors.push([index, this.max]);

docker-compose.yml

@@ -26,6 +26,11 @@ services:
     volumes:
       - ./appsettings.yml:/srv/appsettings.production.yml
     restart: on-failure
+  ping:
+    image: dbogatov/status-site:ping-${DOTNET_TAG}
+    ports:
+      - 8888:8888
+    restart: on-failure
   daemons:
     image: dbogatov/status-site:daemons-${DOTNET_TAG}
     environment:
@@ -34,6 +39,7 @@ services:
       - postgres
     links:
       - "postgres:database"
+      - "ping:ping"
     volumes:
       - ./appsettings.yml:/srv/appsettings.production.yml
     restart: on-failure

ping/Dockerfile

+FROM alpine
+
+# Install CA Certs for HTTPS
+RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
+
+# Create directory for the app source code
+WORKDIR /srv
+
+# Copy the binary
+COPY bin/ping /srv
+
+# Indicate the binary as our entrypoint
+ENTRYPOINT /srv/ping
+
+# Expose your port
+EXPOSE 8888

ping/main/main.go

+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type response struct {
+	URL     string
+	Method  string
+	Timeout int // milliseconds
+
+	Latency       int      `json:",omitempty"` // milliseconds
+	Headers       []string `json:",omitempty"`
+	ContentLength int      `json:",omitempty"`
+	StatusCode    int      `json:",omitempty"`
+
+	Error   string `json:",omitempty"`
+	IsError bool
+}
+
+func main() {
+
+	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+
+		w.Header().Set("Content-Type", "application/json")
+
+		url := r.URL.Query().Get("url")
+		method := r.URL.Query().Get("method")
+		timeout, timeoutConversionErr := strconv.Atoi(r.URL.Query().Get("timeout"))
+
+		response := response{
+			URL:     url,
+			Method:  method,
+			Timeout: timeout,
+			IsError: false,
+		}
+
+		if timeoutConversionErr != nil {
+			sendError(
+				&w,
+				fmt.Errorf("%q is not a valid timeout value", r.URL.Query().Get("timeout")),
+				response,
+			)
+			return
+		}
+
+		start := time.Now()
+
+		var resp *http.Response
+		var httpErr error
+		client := http.Client{
+			Timeout: time.Duration(timeout * int(time.Millisecond)),
+		}
+
+		switch strings.ToUpper(method) {
+		case "HEAD":
+			resp, httpErr = client.Head(url)
+		case "GET":
+			resp, httpErr = client.Get(url)
+		default:
+			sendError(&w, fmt.Errorf("Method %q is not supported", method), response)
+			return
+		}
+
+		if httpErr != nil {
+			errorMessage := httpErr.Error()
+
+			if _, ok := httpErr.(*net.OpError); ok {
+				errorMessage = fmt.Sprintf("Can't access %s", url)
+			}
+
+			if _, ok := httpErr.(net.Error); ok && httpErr.(net.Error).Timeout() {
+				errorMessage = "Timeout"
+			}
+
+			sendError(
+				&w,
+				fmt.Errorf(errorMessage),
+				response,
+			)
+			return
+		}
+
+		end := time.Since(start)
+
+		defer resp.Body.Close()
+
+		response.Latency = int(end.Nanoseconds() / 1000000)
+
+		copied, _ := io.Copy(ioutil.Discard, resp.Body)
+		response.ContentLength = int(copied)
+
+		response.StatusCode = resp.StatusCode
+
+		if resp.StatusCode != 200 {
+			sendError(
+				&w,
+				fmt.Errorf("Server responded %d", resp.StatusCode),
+				response,
+			)
+			return
+		}
+
+		response.Headers = make([]string, len(resp.Header))
+
+		for k, v := range resp.Header {
+			response.Headers = append(response.Headers, fmt.Sprintf("%s: %s", k, v))
+		}
+
+		data, _ := json.Marshal(response)
+
+		w.Write(data)
+	})
+
+	log.Fatal(http.ListenAndServe(":8888", nil))
+}
+
+func sendError(w *http.ResponseWriter, error error, response response) {
+	response.IsError = true
+	response.Error = error.Error()
+	response.StatusCode = 502
+
+	data, _ := json.Marshal(response)
+
+	(*w).Write(data)
+}

src/appsettings.development.yml

@@ -5,9 +5,11 @@
     TrackingId: "UA-XXXXXXX-X"
 Data:
   PingSettings:
+    - ServerUrl: "https://socialimps.dbogatov.org"
     - ServerUrl: "https://google.com"
       MaxFailures: 3
       GetMethodRequired: false
+  PingServerUrl: http://localhost:8888
 ServiceManager:
   PingService:
     Enabled: true

src/appsettings.yml

@@ -50,6 +50,7 @@ Data:
     Compilation: "Compilations"
     Log: "Log Messages"
     Ping: "Response time"
+  PingServerUrl: http://ping:8888
 Logging:
   MinLogLevel: "Information"
   LogSeverityReported: "Error"

src/daemons/Program.cs

@@ -51,7 +51,15 @@ namespace StatusMonitor.Daemons
 
 			services.RegisterSharedServices(env, configuration);
 
+			if (env.IsProduction())
+			{
+				services.AddScoped<IPingService, RemotePingService>();
+			}
+			else
+			{
 				services.AddScoped<IPingService, PingService>();
+			}
+
 			services.AddScoped<ICacheService, CacheService>();
 			services.AddScoped<ICleanService, CleanService>();
 			services.AddScoped<IDemoService, DemoService>();

src/daemons/Services/CacheService.cs

@@ -222,18 +222,18 @@ namespace StatusMonitor.Daemons.Services
 							.PingDataPoints
 							.Where(dp => dp.Metric == metric)
 							.OrderByDescending(dp => dp.Timestamp)
-							.Select(dp => dp.HttpStatusCode)
+							.Select(dp => dp.Success)
 							.ToListAsync());
 
 					if (
-						pingValues.Take(10).Count(code => code != System.Net.HttpStatusCode.OK.AsInt())
+						pingValues.Take(10).Count(code => !code)
 						>=
 						pingSetting.MaxFailures
 					)
 					{
 						label = AutoLabels.Critical;
 					}
-					else if (pingValues.First() != System.Net.HttpStatusCode.OK.AsInt())
+					else if (!pingValues.First())
 					{
 						label = AutoLabels.Warning;
 					}

src/daemons/Services/DemoService.cs

@@ -81,7 +81,7 @@ namespace StatusMonitor.Daemons.Services
 					{
 						Timestamp = timestamp ?? DateTime.UtcNow,
 						Metric = metric,
-						HttpStatusCode = success ? HttpStatusCode.OK.AsInt() : HttpStatusCode.ServiceUnavailable.AsInt(),
+						Success = success,
 						ResponseTime = new TimeSpan(0, 0, 0, 0, success ? random.Next(100, 900) : 0)
 					};
 					await _context.PingDataPoints.AddAsync((PingDataPoint)result);

src/daemons/Services/DiscrepancyService.cs

@@ -310,7 +310,7 @@ namespace StatusMonitor.Daemons.Services
 			var failures = pings
 				.Select(p => new
 				{
-					StatusOK = p.HttpStatusCode == HttpStatusCode.OK.AsInt(),
+					StatusOK = p.Success,
 					Timestamp = p.Timestamp
 				});
 
@@ -473,7 +473,7 @@ namespace StatusMonitor.Daemons.Services
 									dp.Metric.Type == discrepancy.MetricType.AsInt() &&
 									dp.Timestamp > discrepancy.DateFirstOffense
 								)
-								.AnyAsync(dp => dp.HttpStatusCode == HttpStatusCode.OK.AsInt())
+								.AnyAsync(dp => dp.Success)
 						)
 						{
 							resolvedDiscrepancies.Add(discrepancy);

src/daemons/Services/PingService.cs

@@ -8,6 +8,10 @@ using System.Net;
 using Microsoft.Extensions.Logging;
 using StatusMonitor.Shared.Services;
 using StatusMonitor.Shared.Services.Factories;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
+using System.Collections.Generic;
+using Microsoft.AspNetCore.WebUtilities;
 
 namespace StatusMonitor.Daemons.Services
 {
@@ -26,6 +30,94 @@ namespace StatusMonitor.Daemons.Services
 		Task<PingDataPoint> PingServerAsync(PingSetting setting);
 	}
 
+	/// <summary>
+	/// Makes requests to a separate ping server
+	/// Expects RemotePingServerResponse response
+	/// </summary>
+	public class RemotePingService : IPingService
+	{
+		private readonly IMetricService _metrics;
+		private readonly ILogger<PingService> _logger;
+		private readonly IConfiguration _conf;
+		private readonly IHttpClientFactory _factory;
+
+		public RemotePingService(
+			IMetricService metrics,
+			ILogger<PingService> logger,
+			IConfiguration conf,
+			IHttpClientFactory factory
+		)
+		{
+			_conf = conf;
+			_logger = logger;
+			_metrics = metrics;
+			_factory = factory;
+		}
+
+		public async Task<PingDataPoint> PingServerAsync(PingSetting setting)
+		{
+			using (var client = _factory.BuildClient())
+			{
+				var parameters = new Dictionary<string, string> {
+					{ "url", setting.ServerUrl },
+					{ "method", setting.GetMethodRequired ? "GET" : "HEAD" },
+					{ "timeout", setting.MaxResponseTime.TotalMilliseconds.ToString() }
+				};
+
+				var result = await client.GetAsync(QueryHelpers.AddQueryString(_conf["Data:PingServerUrl"], parameters));
+
+				var data = JsonConvert.DeserializeObject<RemotePingServerResponse>(
+					await result.Content.ReadAsStringAsync()
+				);
+
+				_logger.LogDebug(LoggingEvents.Ping.AsInt(), $"Ping completed for {setting.ServerUrl}");
+
+				var metric = await _metrics.GetOrCreateMetricAsync(Metrics.Ping, new Uri(setting.ServerUrl).Host);
+
+				return
+					data.IsError ?
+					new PingDataPoint
+					{
+						Metric = metric,
+						ResponseTime = new TimeSpan(0),
+						Success = data.StatusCode / 100 == 2, // 2xx
+						Message = data.Error
+					}:
+					new PingDataPoint
+					{
+						Metric = metric,
+						ResponseTime = new TimeSpan(0, 0, 0, 0, data.Latency),
+						Success = data.StatusCode / 100 == 2, // 2xx
+						Message = "OK"
+					};
+			}
+		}
+	}
+
+	/// <summary>
+	/// Structure of a separate ping server response expected by RemotePingService
+	/// </summary>
+	internal class RemotePingServerResponse
+	{
+		public string Url { get; set; }
+		public string Method { get; set; }
+		/// <summary>
+		/// In milliseconds
+		/// </summary>
+		public int Timeout { get; set; }
+
+		/// <summary>
+		/// In milliseconds
+		/// </summary>
+		public int Latency { get; set; }
+		public string[] Headers { get; set; }
+		public int ContentLength { get; set; }
+		public int StatusCode { get; set; }
+
+		public string Error { get; set; }
+		public bool IsError { get; set; }
+	}
+
 	public class PingService : IPingService
 	{
 		private readonly IMetricService _metrics;
@@ -98,7 +190,8 @@ namespace StatusMonitor.Daemons.Services
 				{
 					Metric = metric,
 					ResponseTime = responseTime,
-					HttpStatusCode = statusCode.AsInt()
+					Success = statusCode.AsInt() / 100 == 2, // 2xx
+					Message = statusCode.AsInt() / 100 == 2 ? "OK" : "Failure"
 				};
 			}
 		}

src/shared/Models/Entities/DataPoints.cs

@@ -117,12 +117,13 @@ namespace StatusMonitor.Shared.Models.Entities
 		/// 0 for ServiceUnavailable status code.
 		/// </summary>
 		public TimeSpan ResponseTime { get; set; }
-		public int HttpStatusCode { get; set; }
+		public bool Success { get; set; }
+		public string Message { get; set; }
 
 		public override int? NormalizedValue()
 		{
 			return
-				HttpStatusCode == System.Net.HttpStatusCode.OK.AsInt() ? 
+				Success ?
 				Convert.ToInt32(ResponseTime.TotalMilliseconds) :
 				(int?)null
 			;
@@ -134,7 +135,8 @@ namespace StatusMonitor.Shared.Models.Entities
 			{
 				Timestamp,
 				ResponseTime = Convert.ToInt32(ResponseTime.TotalMilliseconds),
-				HttpStatusCode
+				Success,
+				Message
 			};
 		}
 	}

test/Mock/ResponseHandler.cs

@@ -15,16 +15,16 @@ namespace StatusMonitor.Tests.Mock
 	public class ResponseHandler : DelegatingHandler
 	{
 		private readonly Dictionary<Uri, HttpResponseOption> _urls = new Dictionary<Uri, HttpResponseOption>();
-		private readonly Dictionary<Uri, Action> _actions = new Dictionary<Uri, Action>();
+		private readonly Dictionary<string, Func<string>> _actions = new Dictionary<string, Func<string>>();
 
-		public void AddAction(Uri uri, Action action)
+		public void AddAction(Uri uri, Func<string> action)
 		{
-			_actions.Add(uri, action);
+			_actions.Add(uri.Host, action);
 		}
 
 		public void RemoveAction(Uri uri)
 		{
-			_actions.Remove(uri);
+			_actions.Remove(uri.Host);
 		}
 
 		public void AddHandler(Uri uri, HttpResponseOption option)
@@ -44,10 +44,12 @@ namespace StatusMonitor.Tests.Mock
 		{
 			if (_actions.Count > 0)
 			{
-				if (_actions.ContainsKey(request.RequestUri))
+				if (_actions.ContainsKey(request.RequestUri.Host))
 				{
-					_actions[request.RequestUri].Invoke();
-					return await Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK));
+					return new HttpResponseMessage {
+						Content = new StringContent(_actions[request.RequestUri.Host].Invoke()),
+						StatusCode = HttpStatusCode.OK
+					};
 				}
 				else
 				{
@@ -78,7 +80,7 @@ namespace StatusMonitor.Tests.Mock
 				}
 			}
 
-			throw new InvalidOperationException("Mo option or action was registered");
+			throw new InvalidOperationException("No option or action was registered");
 		}
 	}