I’m in the midst of setting Redacted Financial Services Inc. up to use Octopus Deploy. An issue we’ve had with automated deployments in the past is that the tooling reports that everything is okay, but later we find out the software is misconfigured for the environment it’s installed in. To resolve this problem we’ve started including a service health api into our deployed web services & sites. At the end of any deployment we can issue a GET to the api and have the site tell us whether or not it is at least configured in such a way that it can access its dependent databases and services. Anything other than a 200 and we fail the deployment. We are also handing this api call off to Nagios for monitoring.
The code sample below demonstrates the implementation of this api. We deliver as a code-based add-in using our internal nuget feed. It includes health checks for Sql Server , Couchbase , Rabbit MQ , and standard REST services. This code is longish and we’ll probably open source it after we’ve had a chance to bake it. If you try it out let me know how it works for you.
public class ServiceHealthStatus
{
public string Type { get; set; }
public string Status { get; set; }
public string[] Messages { get; set; }
public string Name { get; set; }
public string Data { get; set; }
}
public class ServiceHealthController : ApiController
{
// GET api/ServiceHealth
[Route("api/v1/ServiceHealth")]
[ResponseType(typeof(ServiceHealthStatus))]
public IHttpActionResult Get()
{
var health = GetServiceHealth().ToArray();
if (health.Any(row => row.Status.ToLower() != "up"))
{
var response = Request.CreateResponse(HttpStatusCode.ExpectationFailed, health);
return ResponseMessage(response);
}
return Ok(health);
}
private IEnumerable<ServiceHealthStatus> GetServiceHealth()
{
var results = new List<ServiceHealthStatus>();
var viewModel = new AboutViewModel();
try
{
// these foreach blocks are deliberately NOT LINQ-ified so that
// if an exception occurs we'll still have the first items from each
// collection in the results list.
//
var connectionStringHealth = GetConnectionStringHealth(viewModel);
foreach (var result in connectionStringHealth)
{
results.Add(result);
}
var serviceEndpointHealth = GetServiceEndpointHealth(viewModel);
foreach (var result in serviceEndpointHealth)
{
results.Add(result);
}
var rabbitHealth = GetRabbitHealth(viewModel);
foreach (var result in rabbitHealth)
{
results.Add(result);
}
var couchbaseHealth = GetCouchbaseHealth(viewModel);
foreach (var result in couchbaseHealth)
{
results.Add(result);
}
}
catch (Exception e)
{
results.Add(new ServiceHealthStatus()
{
Type = "Unknown",
Status = "error",
Messages = new[] {e.Message},
Name = "Exception",
});
}
return results;
}
private IEnumerable<ServiceHealthStatus> GetCouchbaseHealth(AboutViewModel viewModel)
{
foreach (var url in viewModel.Couchbase.Servers.Urls)
{
var healthStatusType = "Couchbase";
var name = "Couchbase";
yield return GetServiceHealthStatusForUrl(url, name, healthStatusType);
}
}
private IEnumerable<ServiceHealthStatus> GetRabbitHealth(AboutViewModel viewModel)
{
foreach (var url in viewModel.Rabbit.Urls)
{
var healthStatusType = "Rabbit";
var name = "Rabbit";
var httpGetUrl = url.Replace("amqp", "http").Replace("5672", "15672"); // use the web client port
yield return GetServiceHealthStatusForUrl(httpGetUrl, name, healthStatusType);
}
}
private static IEnumerable<ServiceHealthStatus> GetServiceEndpointHealth(AboutViewModel viewModel)
{
foreach (var endpoint in viewModel.ServiceEndpoints)
{
var healthStatusType = "Service";
var url = endpoint.Link;
var name = endpoint.Name;
yield return GetServiceHealthStatusForUrl(url, name, healthStatusType);
}
}
private static ServiceHealthStatus GetServiceHealthStatusForUrl(string url, string name, string healthStatusType)
{
Exception exception = null;
HttpResponseMessage response = null;
try
{
var uri = new Uri(url, UriKind.Absolute);
var client = new HttpClient();
response = client.GetAsync(uri).Result;
}
catch (Exception e)
{
exception = e;
}
var status = (exception == null && response != null && response.StatusCode == HttpStatusCode.OK)
? "up"
: "down";
var messages = new List<string>();
if (response != null && response.StatusCode != HttpStatusCode.Accepted)
{
var content = response.Content.ReadAsStringAsync().Result;
messages.Add(content);
}
while (exception != null)
{
messages.Add(exception.Message);
exception = exception.InnerException;
}
return new ServiceHealthStatus()
{
Name = name,
Data = url,
Status = status,
Type = healthStatusType,
Messages = messages.ToArray(),
};
}
private static IEnumerable<ServiceHealthStatus> GetConnectionStringHealth(AboutViewModel viewModel)
{
foreach (var connectionString in viewModel.ConnectionStrings.Cast<ConnectionStringSettings>())
{
Exception exception = null;
var str = connectionString.ConnectionString.Replace("Connect Timeout=30", "Connect Timeout=5");
using (var connection = new SqlConnection(str))
{
try
{
connection.Open();
}
catch (Exception e)
{
exception = e;
}
var status = (exception == null) ? "up" : "down";
var messages = (exception == null) ? new string[] {} : new string[] {exception.Message};
yield return new ServiceHealthStatus()
{
Type = "Database",
Name = connectionString.Name,
Data = connectionString.ConnectionString,
Status = status,
Messages = messages
};
}
}
}
}
// Comment out the line below if your project does not use Couchbase.
#define Couchbase
#if Couchbase
// If this line is giving you a compilation error because your app
// does not use Couchbase,
// Comment out the Couchbase preprocessor directive at the top of this file.
using Couchbase.Configuration.Client.Providers;
#endif
public class AboutViewModel
{
private static ICollection<KeyValuePair<string, string>> GetApplicationConfigurationValues(string sectionName)
{
var section = (NameValueCollection)ConfigurationManager.GetSection(sectionName);
if (section != null)
return
section.AllKeys.Select(key => new KeyValuePair<string, string>(key, section[key])).ToList();
Debug.WriteLine("Unable to find '{0}' section in configuration file.", sectionName);
return new List<KeyValuePair<string, string>>();
}
private ServiceEndpointViewModel MapServiceEndpoint(KeyValuePair<string, string> s)
{
return new ServiceEndpointViewModel()
{
Name = s.Key,
Location = s.Value,
Link = this.GetDocumentationUrl(s.Value)
};
}
private string GetDocumentationUrl(string value)
{
if (string.IsNullOrWhiteSpace(value) || value.EndsWith(".svc") ||
!Uri.IsWellFormedUriString(value, UriKind.Absolute))
return value;
return string.Format("http://{0}/swagger", new Uri(value).Host);
}
private static AuthorizationConfigurationViewModel MapAuthorization(KeyValuePair<string, string> s)
{
string str = string.IsNullOrWhiteSpace(s.Value) ? "{None Defined}" : s.Value;
return new AuthorizationConfigurationViewModel()
{
Name = s.Key,
Groups = str
};
}
private List<LoggerConfigurationViewModel> GetLoggingConfiguration()
{
var allRepositories = LogManager.GetAllRepositories();
var loggers = allRepositories
.SelectMany(r => r.GetCurrentLoggers()
.Cast<Logger>())
.Where(l => l.Appenders.Cast<IAppender>().Any())
.Select(l => this.MapLogger(l))
.ToList()
;
var loggersFromHierarchies = allRepositories
.Cast<Hierarchy>()
.Where(h => h.Root != null)
.Select(h => this.MapLogger(h.Root, "Root"))
.ToList()
;
loggers.AddRange(loggersFromHierarchies);
return loggers;
}
private LoggerConfigurationViewModel MapLogger(Logger l, string name = null)
{
return new LoggerConfigurationViewModel()
{
Name = name ?? l.Name,
LoggingLevel = l.EffectiveLevel.DisplayName,
Appenders = this.MapAppenderCollection(l.Appenders)
};
}
private List<LogAppenderConfigurationViewModel> MapAppenderCollection(AppenderCollection appenderCollection)
{
return appenderCollection.Cast<AppenderSkeleton>().Select(this.MapAppender).ToList();
}
private LogAppenderConfigurationViewModel MapAppender(AppenderSkeleton a)
{
return new LogAppenderConfigurationViewModel()
{
Name = a.Name,
Type = a.GetType().FullName,
LayoutName = a.Layout.GetType().FullName,
Properties = GetProperties(a),
LayoutProperties = GetProperties(a.Layout)
};
}
private static Dictionary<string, object> GetProperties<T>(T item)
{
Type type = item.GetType();
PropertyInfo[] properties = type.GetProperties();
Func<PropertyInfo, object> elementSelector =
p => type.GetProperty(p.Name).GetValue((object)(T)item);
return properties.ToDictionary(p => p.Name, elementSelector);
}
public AboutViewModel()
{
ApplicationName = this.GetType().Assembly.GetName().Name;
AppSettings = ConfigurationManager.AppSettings;
ConnectionStrings = ConfigurationManager.ConnectionStrings;
Assemblies = AssemblyViewModel.FromAssembly(Assembly.GetExecutingAssembly());
ServiceEndpoints = GetApplicationConfigurationValues("service.addresses").Select(this.MapServiceEndpoint).ToList();
Authorizations = GetApplicationConfigurationValues("authorization").Select(MapAuthorization).ToList();
LoggingConfiguration = GetLoggingConfiguration();
Rabbit = RabbitViewModel.From(GetApplicationConfigurationValues("rabbit"));
Couchbase = CouchbaseConfigurationViewModel.FromAssembly(Assembly.GetExecutingAssembly());
}
public string ApplicationName { get; set; }
public string ApplicationSetName { get; set; }
public IEnumerable<AssemblyViewModel> Assemblies { get; set; }
public IEnumerable<ServiceEndpointViewModel> ServiceEndpoints { get; set; }
public IEnumerable<AuthorizationConfigurationViewModel> Authorizations { get; set; }
public IEnumerable<LoggerConfigurationViewModel> LoggingConfiguration { get; set; }
public NameValueCollection AppSettings { get; set; }
public ConnectionStringSettingsCollection ConnectionStrings { get; set; }
public RabbitViewModel Rabbit { get; set; }
public CouchbaseConfigurationViewModel Couchbase { get; set; }
}
public class AssemblyViewModel
{
public static IEnumerable<AssemblyViewModel> FromAssembly(Assembly assembly)
{
var dependentAssemblies = GetDependentAssemblies(assembly);
var viewModels = dependentAssemblies
.Select(a => new AssemblyViewModel
{
AssemblyName = a.Name,
AssemblyVersion = a.Version.ToString()
})
.OrderBy(a => a.AssemblyName);
return viewModels;
}
private static IEnumerable<AssemblyName> GetDependentAssemblies(Assembly assembly)
{
var list = new List<AssemblyName>();
var codebase = new Uri(assembly.CodeBase);
var binPath = codebase.LocalPath;
var binFolder = Path.GetDirectoryName(binPath);
if (binFolder != null)
{
var files = Directory.GetFiles(binFolder, "*.dll");
list.AddRange(files.Select(Assembly.LoadFrom).Select(a => a.GetName()));
}
return list;
}
public string AssemblyName { get; set; }
public string AssemblyVersion { get; set; }
}
public class AuthorizationConfigurationViewModel
{
public string Name { get; set; }
public string Groups { get; set; }
}
public class LoggerConfigurationViewModel
{
public string Name { get; set; }
public string LoggingLevel { get; set; }
public List<LogAppenderConfigurationViewModel> Appenders { get; set; }
}
public class LogAppenderConfigurationViewModel
{
public string Name { get; set; }
public string Type { get; set; }
public string LayoutName { get; set; }
public Dictionary<string, object> Properties { get; set; }
public Dictionary<string, object> LayoutProperties { get; set; }
}
public class RabbitViewModel
{
public RabbitViewModel()
{
Urls = new string[] { };
}
public string[] Urls { get; set; }
public static RabbitViewModel From(ICollection<KeyValuePair<string, string>> rabbitSection)
{
const string key = "RabbitMqServerUrl";
if (!rabbitSection.Any(row => row.Key == key))
{
return new RabbitViewModel();
}
var rawValue = rabbitSection.First(row => row.Key == key).Value;
if (string.IsNullOrWhiteSpace(rawValue))
{
return new RabbitViewModel();
}
var urls = rawValue
.Split('|')
.Select(row => row.Trim())
.Select(row =>
{
var expression = @"amqp:\/\/\w+:";
if (Regex.IsMatch(row, expression))
{
var match = Regex.Match(row, expression);
var startOfPassword = match.Length;
var endOfPassword = row.IndexOf("@");
var charArray = row.ToCharArray();
for (var i = startOfPassword; i < endOfPassword; i++)
{
charArray[i] = '*';
}
var result = new string(charArray);
return result;
}
return row;
})
.ToArray()
;
return new RabbitViewModel()
{
Urls = urls,
};
}
}
public class ServiceEndpointViewModel
{
public string Name { get; set; }
public string Location { get; set; }
public string Link { get; set; }
}
[XmlRoot(ElementName = "couchbase")]
public class CouchbaseConfigurationViewModel
{
public CouchbaseConfigurationViewModel()
{
Buckets = new List<CouchbaseBucketViewModel>();
ConnectionPool = new CouchbaseConnectionPoolViewModel();
Servers = new CouchbaseServersViewModel()
{
Urls = new List<string>(),
};
}
[XmlAttribute("useSsl")]
public bool UseSsl { get; set; }
[XmlElement("servers")]
public CouchbaseServersViewModel Servers { get; set; }
public List<CouchbaseBucketViewModel> Buckets { get; set; }
#if Couchbase
public static CouchbaseConnectionPoolViewModel Create(ConnectionPoolElement element)
{
return new CouchbaseConnectionPoolViewModel()
{
Name = element.Name,
MaxSize = element.MaxSize,
MinSize = element.MinSize,
SendTimeout = element.SendTimeout,
};
}
public static CouchbaseBucketViewModel Create(BucketElement row)
{
return new CouchbaseBucketViewModel()
{
UseSsl = row.UseSsl,
OperationLifespan = row.OperationLifespan,
Name = row.Name,
ConnectionPool = Create(row.ConnectionPool),
ObserveTimeout = row.ObserveTimeout,
ObserveInterval = row.ObserveInterval,
UseEnhancedDurability = row.UseEnhancedDurability,
};
}
#endif
public static CouchbaseConfigurationViewModel FromAssembly(Assembly assembly)
{
#if Couchbase
var section = ConfigurationManager.GetSection("couchbase") as CouchbaseClientSection;
var servers = section.Servers
.Cast<UriElement>()
.Select(row =>
{
return row.Uri.OriginalString;
})
.ToList()
;
var buckets = section.Buckets
.Cast<BucketElement>()
.Select(Create)
.ToList();
var result = new CouchbaseConfigurationViewModel()
{
ConnectionPool = Create(section.ConnectionPool),
Servers = new CouchbaseServersViewModel()
{
Urls = servers,
},
Buckets = buckets,
UseSsl = section.UseSsl,
ApiPort = section.ApiPort,
DirectPort = section.DirectPort,
EnableConfigHeartBeat = section.EnableConfigHeartBeat,
EnableOperationTiming = section.EnableOperationTiming,
EnableTcpKeepAlives = section.EnableTcpKeepAlives,
Expect100Continue = section.Expect100Continue
};
return result;
#else
return new CouchbaseConfigurationViewModel();
#endif
}
public bool Expect100Continue { get; set; }
public bool EnableTcpKeepAlives { get; set; }
public bool EnableOperationTiming { get; set; }
public bool EnableConfigHeartBeat { get; set; }
public int DirectPort { get; set; }
public int ApiPort { get; set; }
public CouchbaseConnectionPoolViewModel ConnectionPool { get; set; }
}
public class CouchbaseBucketViewModel
{
[XmlAttribute("name")]
public string Name { get; set; }
[XmlAttribute("useSsl")]
public bool UseSsl { get; set; }
[XmlAttribute("password")]
public string Password { get; set; }
[XmlAttribute("openLifespan")]
public uint? OperationLifespan { get; set; }
[XmlElement("connectionPool")]
public CouchbaseConnectionPoolViewModel ConnectionPool { get; set; }
public int ObserveTimeout { get; set; }
public int ObserveInterval { get; set; }
public bool UseEnhancedDurability { get; set; }
}
public class CouchbaseConnectionPoolViewModel
{
public string Name { get; set; }
public long MaxSize { get; set; }
public long MinSize { get; set; }
public long SendTimeout { get; set; }
}
public class CouchbaseServersViewModel
{
public string Bucket { get; set; }
public string BucketPassword { get; set; }
public List<string> Urls { get; set; }
}Like this: Like Loading...
