Import from ntp.exactlywww.com

This commit is contained in:
Carsten Larsen 2016-12-21 23:23:43 +01:00
parent 994af48594
commit 42a9895e43
12 changed files with 9706 additions and 82 deletions

View File

@ -0,0 +1,61 @@
//
// Copyright (c) 2013-2016 Carsten Sonne Larsen <cs@innolan.dk>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using System;
using Newtonsoft.Json;
using Ntp.Analyzer.Data.Log;
using Ntp.Common.Log;
namespace Ntp.Analyzer.Data.Import
{
public sealed class ExactlyAdapter : WebAdapter
{
public ExactlyAdapter(LogBase log)
: base(log)
{
}
protected override string Provider => "ntp.exactlywww.com";
public ExactlyTimeServer ImportServer(int orgId)
{
string url = $"http://ntp.exactlywww.com/ntp/api/?id={orgId}";
string json = FetchHtml(url, orgId);
if (json == null)
return null;
ExactlyTimeServer server;
try
{
server = JsonConvert.DeserializeObject<ExactlyTimeServer>(json);
}
catch (Exception e)
{
Log.TimeServerParseError("json", e);
return null;
}
return server;
}
}
}

View File

@ -0,0 +1,37 @@
//
// Copyright (c) 2013-2016 Carsten Sonne Larsen <cs@innolan.dk>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using Newtonsoft.Json;
namespace Ntp.Analyzer.Data.Import
{
public class ExactlySuccess
{
[JsonProperty("4")]
public string Ip4 { get; set; }
[JsonProperty("6")]
public string Ip6 { get; set; }
[JsonProperty("host")]
public string Host { get; set; }
}
}

View File

@ -0,0 +1,194 @@
//
// Copyright (c) 2013-2016 Carsten Sonne Larsen <cs@innolan.dk>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using System;
using Newtonsoft.Json;
namespace Ntp.Analyzer.Data.Import
{
public class ExactlyTimeServer
{
/// <summary>
/// A link to the server information page at support.ntp.org.
/// </summary>
[JsonProperty("url")]
public string Url { get; set; }
/// <summary>
/// The country code for where the server is located.
/// </summary>
[JsonProperty("iso")]
public string Iso { get; set; }
/// <summary>
/// The physical location of the server, useful in selecting upstream servers.
/// </summary>
[JsonProperty("location")]
public string Location { get; set; }
/// <summary>
/// The organization sponsoring the operation of the server.
/// </summary>
[JsonProperty("sponsor")]
public string Sponsor { get; set; }
/// <summary>
/// The region/area that the server is intended to serve. If you're outside that area,
/// you probably shouldn't be using it, unless there is a shortage of servers in your
/// area.
/// </summary>
[JsonProperty("service_area")]
public string ServiceArea { get; set; }
[JsonProperty("access")]
public string Access { get; set; }
/// <summary>
/// A 1 indicates the server operator would like to be notified, via the details in
/// contact if you are using their server. A zero means they don't care, just happy
/// to be of help.
/// </summary>
[JsonProperty("notify")]
public string Notify { get; set; }
/// <summary>
/// A UNIX timestamp indicating the last time the server page was updated by the owner.
/// </summary>
/// <example>1102697989</example>
[JsonProperty("modified")]
public long ModifiedRaw { get; set; }
/// <summary>
/// Last time the server page was updated by the owner.
/// </summary>
public DateTime Modified => new DateTime(1970, 1, 1, 0, 0, 0, 0).AddSeconds(ModifiedRaw);
/// <summary>
/// The hostname to be used if usedns = 1.
/// </summary>
/// <example>ntp.cpsc.ucalgary.ca</example>
[JsonProperty("hostname")]
public string Hostname { get; set; }
/// <summary>
/// IP (v4) address to be used if usedns = 0.
/// </summary>
/// <example>136.159.2.2</example>
[JsonProperty("ipv4")]
public string Ipv4 { get; set; }
/// <summary>
/// If available, the IP (v6) address to be used if usedns = 0.
/// </summary>
[JsonProperty("ipv6")]
public string Ipv6 { get; set; }
/// <summary>
/// Indicates that downstream servers should use the hostname, not DNS.
/// </summary>
[JsonProperty("usedns")]
public string UseDns { get; set; }
/// <summary>
/// More information about the access granted, or notification requested.
/// </summary>
[JsonProperty("access_details")]
public string AccessDetails { get; set; }
/// <summary>
/// If notify = 1, use this information to contact the server operator.
/// </summary>
/// <example>Brad Arlt (timekeeper@cpsc.ucalgary.ca)</example>
[JsonProperty("contact")]
public string Contact { get; set; }
/// <summary>
/// These are the IP addresses returned by a DNS query for IPv4 (A)
/// records, should not be used directly, use the hostname. Only
/// successful addresses are listed.
/// </summary>
[JsonProperty("dns_a")]
public string[] DnsA { get; set; }
/// <summary>
/// These are the IP addresses returned by a DNS query for IPv6 (AAAA)
/// records, should not be used directly, use the hostname.
/// </summary>
[JsonProperty("dns_aaaa")]
public string[] DnsAaaa { get; set; }
/// <summary>
/// If this is an empty array, that is good. If there is stuff here,
/// something is entered wrong on the server page, or misconfigured
/// with DNS, or possibly only some addresses worked.
/// </summary>
[JsonProperty("status")]
public string[] Status { get; set; }
/// <summary>
/// If the last test was successful, this tells you which test succeeded:
/// hostname, IPv4, or IPv6.
/// </summary>
[JsonProperty("success")]
public ExactlySuccess Success { get; set; }
/// <summary>
/// The numerical ID, as queried from the list at support.ntp.org.
/// </summary>
[JsonProperty("id")]
public int Id { get; set; }
/// <summary>
/// How many times we have successfully contacted this server.
/// </summary>
[JsonProperty("successes")]
public int Successes { get; set; }
/// <summary>
/// How many times we have failed to contact thsi server.
/// </summary>
[JsonProperty("failures")]
public int Failures { get; set; }
/// <summary>
/// Was the last attempt successful? 1 = Yes, 0 = No.
/// </summary>
[JsonProperty("up")]
public int UpRaw { get; set; }
/// <summary>
/// Was the last attempt successful?
/// </summary>
public bool IsUp => UpRaw == 1;
/// <summary>
/// UNIX timestamp of the last time this server was successfully contacted.
/// </summary>
/// <example>1482208125</example>
[JsonProperty("contacted")]
public long ContactedRaw { get; set; }
/// <summary>
/// Last time this server was successfully contacted.
/// </summary>
public DateTime Contacted => new DateTime(1970, 1, 1, 0, 0, 0, 0).AddSeconds(ContactedRaw);
}
}

View File

@ -21,38 +21,33 @@
using System;
using System.Globalization;
using System.IO;
using System.Net;
using Ntp.Analyzer.Data.Log;
using Ntp.Common.Log;
namespace Ntp.Analyzer.Data.Import
{
internal sealed class TimeServerLoader
internal sealed class TimeServerLoader : WebAdapter
{
internal TimeServerLoader(LogBase log)
: base(log)
{
this.log = log;
}
private const string NtpOrg = "http://support.ntp.org/bin/view/Servers/PublicTimeServer";
private const string ServerTagStart = @"<div class=""twikiAfterText""></div><div class=""twikiForm"">";
private const string ServerTagEnd = @"</div><!-- /twikiForm --></div><!-- /patternContent-->";
private readonly LogBase log;
protected override string Provider => "support.ntp.org";
public string ImportServer(int orgId)
{
if (orgId >= 10000)
{
log.WriteLine("Not contacting support.ntp.org for ID > 10000.", Severity.Debug);
return null;
}
string orgString = orgId.ToString(CultureInfo.InvariantCulture).PadLeft(6, '0');
string url = $"http://support.ntp.org/bin/view/Servers/PublicTimeServer{orgString}";
string html = FetchHtml(url, orgId);
string html = GetOrgHtml(orgId);
if (html == null)
{
log.WriteLine("No HTML received for TimeServer.", Severity.Debug);
return null;
}
html = html.Replace("<nos>", string.Empty);
try
{
@ -64,56 +59,9 @@ public string ImportServer(int orgId)
}
catch (Exception e)
{
log.WriteLine("Could not find TimeServer HTML.", Severity.Error);
log.WriteLine(e);
Log.TimeServerParseError("HTML", e);
return null;
}
}
private string GetOrgHtml(int orgId)
{
string url = NtpOrg + orgId.ToString(CultureInfo.InvariantCulture).PadLeft(6, '0');
try
{
var client = new WebClient();
client.Headers.Add(
"user-agent",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2;)"
);
Stream stream = client.OpenRead(url);
log.WriteLine("Fetching data from support.ntp.org for server ID " + orgId, Severity.Info);
if (stream != null)
{
var reader = new StreamReader(stream);
string html = reader.ReadToEnd();
reader.Close();
return html.Replace("<nos>", string.Empty);
}
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && response.StatusCode == HttpStatusCode.NotFound)
{
log.WriteLine("TimeServer HTML page not found. ID: " + orgId, Severity.Warn);
}
else
{
log.WriteLine("Error while contacting support.ntp.org with ID " + orgId, Severity.Error);
}
log.WriteLine(e);
}
catch (Exception e)
{
log.WriteLine("Error while contacting support.ntp.org with ID " + orgId, Severity.Error);
log.WriteLine(e);
}
return null;
}
}
}

View File

@ -0,0 +1,101 @@
//
// Copyright (c) 2013-2016 Carsten Sonne Larsen <cs@innolan.dk>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
using System;
using System.IO;
using System.Net;
using Ntp.Analyzer.Data.Log;
using Ntp.Common.Log;
namespace Ntp.Analyzer.Data.Import
{
public abstract class WebAdapter
{
protected WebAdapter(LogBase log)
{
Log = log;
}
protected readonly LogBase Log;
protected abstract string Provider { get; }
protected string FetchHtml(string url, int orgId)
{
if (orgId >= 5000)
{
Log.TimeServerMaxId(Provider);
return null;
}
string html = Download(url, orgId);
if (html != null)
return html;
Log.TimeServerNotRecieved(orgId);
return null;
}
private string Download(string url, int orgId)
{
try
{
var client = new WebClient();
client.Headers.Add(
"user-agent",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2;)"
);
var stream = client.OpenRead(url);
Log.TimeServerDownload(Provider, url, orgId);
if (stream != null)
{
var reader = new StreamReader(stream);
string html = reader.ReadToEnd();
reader.Close();
return html;
}
}
catch (WebException e)
{
var response = e.Response as HttpWebResponse;
if (response != null && response.StatusCode == HttpStatusCode.NotFound)
{
Log.TimeServerNotFound(Provider, orgId);
}
else
{
Log.TimeServerError(Provider, orgId);
}
Log.WriteLine(e);
}
catch (Exception e)
{
Log.TimeServerError(Provider, orgId);
Log.WriteLine(e);
}
return null;
}
}
}

View File

@ -88,7 +88,42 @@ internal static void UpdateError(this LogBase log, string table, Exception e)
private static void Advice(LogBase log)
{
log.WriteLine($"Try setting 'Create Yes' in configuration.", Severity.Notice);
log.WriteLine("Please check your configuration.", Severity.Notice);
}
}
internal static class LogMessagesImport
{
internal static void TimeServerDownload(this LogBase log, string provider, string url, int orgId)
{
log.WriteLine($"Fetching data from {provider} for server ID {orgId}", Severity.Info);
log.WriteLine($"Downloading ... {url}", Severity.Debug);
}
internal static void TimeServerError(this LogBase log, string provider, int orgId)
{
log.WriteLine($"Error while contacting {provider} with ID {orgId}", Severity.Warn);
}
internal static void TimeServerMaxId(this LogBase log, string provider)
{
log.WriteLine($"Not contacting {provider} for ID > 10000.", Severity.Debug);
}
internal static void TimeServerNotFound(this LogBase log, string provider, int orgId)
{
log.WriteLine($"Time server not found at {provider}. ID: {orgId}", Severity.Info);
}
internal static void TimeServerNotRecieved(this LogBase log, int orgId)
{
log.WriteLine($"Nothing received for time server ID {orgId}.", Severity.Debug);
}
internal static void TimeServerParseError(this LogBase log, string type, Exception e)
{
log.WriteLine($"Could not parse time server {type}.", Severity.Warn);
log.WriteLine(e);
}
}
}

View File

@ -1,7 +1,5 @@
EXTRA_DIST =
# Warning: This is an automatically generated file, do not edit!
EXTRA_DIST =
if ENABLE_DEBUG
ASSEMBLY_COMPILER_COMMAND = mcs
@ -32,6 +30,7 @@ NTP_DATA_PROVIDER_DLL_MDB_SOURCE=../bin/Ntp.Data.Provider.dll.mdb
NTP_DATA_PROVIDER_DLL_MDB=$(BUILD_DIR)/Ntp.Data.Provider.dll.mdb
MYSQL_DATA_DLL_SOURCE=../packages/MySql.Data.6.9.9/lib/net45/MySql.Data.dll
NPGSQL_DLL_SOURCE=../packages/Npgsql.3.1.9/lib/net451/Npgsql.dll
NEWTONSOFT_DLL_SOURCE=../packages/Newtonsoft.Json.9.0.1/lib/net45/Newtonsoft.Json.dll
endif
@ -39,7 +38,7 @@ if ENABLE_RELEASE
ASSEMBLY_COMPILER_COMMAND = mcs
ASSEMBLY_COMPILER_FLAGS = -noconfig -codepage:utf8 -warn:4 -optimize+ "-define:TRACE"
ASSEMBLY = ../bin/Ntp.Analyzer.Data.dll
ASSEMBLY_MDB =
ASSEMBLY_MDB =
COMPILE_TARGET = library
PROJECT_REFERENCES = \
../bin/Ntp.Analyzer.Objects.dll \
@ -59,6 +58,7 @@ NTP_DATA_PROVIDER_DLL_SOURCE=../bin/Ntp.Data.Provider.dll
NTP_DATA_PROVIDER_DLL_MDB=
MYSQL_DATA_DLL_SOURCE=../packages/MySql.Data.6.9.9/lib/net45/MySql.Data.dll
NPGSQL_DLL_SOURCE=../packages/Npgsql.3.1.9/lib/net451/Npgsql.dll
NEWTONSOFT_DLL_SOURCE=../packages/Newtonsoft.Json.9.0.1/lib/net45/Newtonsoft.Json.dll
endif
@ -76,20 +76,24 @@ PROGRAMFILES = \
$(NTP_DATA_PROVIDER_DLL) \
$(NTP_DATA_PROVIDER_DLL_MDB) \
$(MYSQL_DATA_DLL) \
$(NPGSQL_DLL)
$(NPGSQL_DLL) \
$(NEWTONSOFT_DLL)
RESGEN=resgen2
all: $(ASSEMBLY) $(PROGRAMFILES)
all: $(ASSEMBLY) $(PROGRAMFILES)
FILES = \
../Shared/AssemblyInfo.cs \
Changes/Change03.cs \
DataFace.cs \
Import/TimeServers.cs \
Import/ExactlySuccess.cs \
Import/ExactlyTimeServer.cs \
Import/ExactlyAdapter.cs \
Import/TimeServerImporter.cs \
Import/TimeServerLoader.cs \
Import/TimeServers.cs \
Import/WebAdapter.cs \
Log/LogExtensions.cs \
Sql/DriftReadingDatabaseMapper.cs \
Sql/HostDatabaseMapper.cs \
@ -105,13 +109,13 @@ FILES = \
Sql/ReadingBulkMapper.cs \
Sql/AssociationEntryMapper.cs \
Sql/HostIoReadingDatabaseMapper.cs \
Changes/Change02.cs
Changes/Change02.cs
DATA_FILES =
DATA_FILES =
RESOURCES =
RESOURCES =
EXTRAS =
EXTRAS =
REFERENCES = \
System \
@ -119,9 +123,9 @@ REFERENCES = \
System.Core \
System.Xml
DLL_REFERENCES =
DLL_REFERENCES = $(NEWTONSOFT_DLL)
CLEANFILES = $(PROGRAMFILES)
CLEANFILES = $(PROGRAMFILES)
include $(top_srcdir)/Makefile.include
@ -131,10 +135,11 @@ NTP_DATA_DLL = $(BUILD_DIR)/Ntp.Data.dll
NTP_DATA_PROVIDER_DLL = $(BUILD_DIR)/Ntp.Data.Provider.dll
MYSQL_DATA_DLL = $(BUILD_DIR)/MySql.Data.dll
NPGSQL_DLL = $(BUILD_DIR)/Npgsql.dll
NEWTONSOFT_DLL = $(BUILD_DIR)/Newtonsoft.Json.dll
$(eval $(call emit-deploy-target,MYSQL_DATA_DLL))
$(eval $(call emit-deploy-target,NPGSQL_DLL))
$(eval $(call emit-deploy-target,NEWTONSOFT_DLL))
$(eval $(call emit_resgen_targets))
$(build_xamlg_list): %.xaml.g.cs: %.xaml

View File

@ -30,6 +30,10 @@
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="Newtonsoft.Json, Version=9.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System" />
<Reference Include="System.Data" />
<Reference Include="System.Core" />
@ -39,9 +43,13 @@
<Compile Include="..\Shared\AssemblyInfo.cs" />
<Compile Include="Changes\Change03.cs" />
<Compile Include="DataFace.cs" />
<Compile Include="Import\TimeServers.cs" />
<Compile Include="Import\ExactlySuccess.cs" />
<Compile Include="Import\ExactlyTimeServer.cs" />
<Compile Include="Import\ExactlyAdapter.cs" />
<Compile Include="Import\TimeServerImporter.cs" />
<Compile Include="Import\TimeServerLoader.cs" />
<Compile Include="Import\TimeServers.cs" />
<Compile Include="Import\WebAdapter.cs" />
<Compile Include="Log\LogExtensions.cs" />
<Compile Include="Sql\DriftReadingDatabaseMapper.cs" />
<Compile Include="Sql\HostDatabaseMapper.cs" />
@ -60,7 +68,9 @@
<Compile Include="Changes\Change02.cs" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
<ItemGroup />
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Ntp.Analyzer.Objects\Ntp.Analyzer.Objects.csproj">
<Project>{02912378-E62D-4445-BA30-F56D3ABE9DA2}</Project>

View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Newtonsoft.Json" version="9.0.1" targetFramework="net452" />
</packages>

Binary file not shown.

Binary file not shown.

File diff suppressed because it is too large Load Diff