本文講解基於kestrel開發類似Fiddler應用的過程，讓讀者瞭解kestrel網路程式設計裡面的kestrel中介軟體和http應用中介軟體。由於最終目的不是輸出完整功能的產品，所以這裡只實現Fiddler最核心的http請求和響應內容檢視的功能。本文章是KestrelApp專案裡面的一個demo的講解，希望對您有用。
所謂傳輸層，其目的是為了讓應用協定資料安全、可靠、快速等傳輸而存在的一種協定，其特徵是把應用協定的報文做為自己的負載，常見的tcp、udp、quic、tls等都可以理解為傳輸層協定。
比如http協定，常見有如下的傳輸方式：
http over tcphttp over tls over tcphttp over quic over udpFiddler要處理以下三種http傳輸情況：
http over tcp：直接http請求首頁http over proxy over tcp：代理http流量http over tls over proxy over tcp：代理https流量kestrel目前的傳輸層基於tcp或quic兩種，同時內建了tls中介軟體，需要呼叫ListenOptions.UseHttps()來使用tls中介軟體。kestrel的中介軟體的表現形式為：Func<ConnectionDelegate, ConnectionDelegate>，為了使用讀者能夠簡單理解中介軟體，我在KestrelFramework裡定義了kestrel中介軟體的變種介面，大家基於此介面來實現更多的中介軟體就方便很多：
/// <summary>
/// Kestrel的中介軟體介面
/// </summary>
public interface IKestrelMiddleware
{
/// <summary>
/// 執行
/// </summary>
/// <param name="next"></param>
/// <param name="context"></param>
/// <returns></returns>
Task InvokeAsync(ConnectionDelegate next, ConnectionContext context);
}
Filddler最基礎的功能是它是一個http代理伺服器， 我們需要為kestrel編寫代理中介軟體，用於處理代理傳輸層。http代理協定分兩種：普通的http代理和Connect隧道代理。兩種的報文者是遵循http1.0或1.1的文字格式，我們可以使用kestrel自帶的HttpParser<>來解析這些複雜的http文字協定。
在中介軟體程式設計模式中，Feature是一個很重要的中介軟體溝通橋樑，它往往是某個中介軟體工作之後，留下的財產，讓之後的中介軟體來獲取並受益。我們的代理中介軟體，也設計了IProxyFeature，告訴之後的中介軟體一些代理特徵。
/// <summary>
/// 代理Feature
/// </summary>
public interface IProxyFeature
{
/// <summary>
/// 代理主機
/// </summary>
HostString ProxyHost { get; }
/// <summary>
/// 代理協定
/// </summary>
ProxyProtocol ProxyProtocol { get; }
}
/// <summary>
/// 代理協定
/// </summary>
public enum ProxyProtocol
{
/// <summary>
/// 無代理
/// </summary>
None,
/// <summary>
/// http代理
/// </summary>
HttpProxy,
/// <summary>
/// 隧道代理
/// </summary>
TunnelProxy
}
/// <summary>
/// 代理中介軟體
/// </summary>
sealed class KestrelProxyMiddleware : IKestrelMiddleware
{
private static readonly HttpParser<HttpRequestHandler> httpParser = new();
private static readonly byte[] http200 = Encoding.ASCII.GetBytes("HTTP/1.1 200 Connection Established\r\n\r\n");
private static readonly byte[] http400 = Encoding.ASCII.GetBytes("HTTP/1.1 400 Bad Request\r\n\r\n");
/// <summary>
/// 解析代理
/// </summary>
/// <param name="next"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task InvokeAsync(ConnectionDelegate next, ConnectionContext context)
{
var input = context.Transport.Input;
var output = context.Transport.Output;
var request = new HttpRequestHandler();
while (context.ConnectionClosed.IsCancellationRequested == false)
{
var result = await input.ReadAsync();
if (result.IsCanceled)
{
break;
}
try
{
if (ParseRequest(result, request, out var consumed))
{
if (request.ProxyProtocol == ProxyProtocol.TunnelProxy)
{
input.AdvanceTo(consumed);
await output.WriteAsync(http200);
}
else
{
input.AdvanceTo(result.Buffer.Start);
}
context.Features.Set<IProxyFeature>(request);
await next(context);
break;
}
else
{
input.AdvanceTo(result.Buffer.Start, result.Buffer.End);
}
if (result.IsCompleted)
{
break;
}
}
catch (Exception)
{
await output.WriteAsync(http400);
break;
}
}
}
/// <summary>
/// 解析http請求
/// </summary>
/// <param name="result"></param>
/// <param name="request"></param>
/// <param name="consumed"></param>
/// <returns></returns>
private static bool ParseRequest(ReadResult result, HttpRequestHandler request, out SequencePosition consumed)
{
var reader = new SequenceReader<byte>(result.Buffer);
if (httpParser.ParseRequestLine(request, ref reader) &&
httpParser.ParseHeaders(request, ref reader))
{
consumed = reader.Position;
return true;
}
else
{
consumed = default;
return false;
}
}
/// <summary>
/// 代理請求處理器
/// </summary>
private class HttpRequestHandler : IHttpRequestLineHandler, IHttpHeadersHandler, IProxyFeature
{
private HttpMethod method;
public HostString ProxyHost { get; private set; }
public ProxyProtocol ProxyProtocol
{
get
{
if (ProxyHost.HasValue == false)
{
return ProxyProtocol.None;
}
if (method == HttpMethod.Connect)
{
return ProxyProtocol.TunnelProxy;
}
return ProxyProtocol.HttpProxy;
}
}
void IHttpRequestLineHandler.OnStartLine(HttpVersionAndMethod versionAndMethod, TargetOffsetPathLength targetPath, Span<byte> startLine)
{
method = versionAndMethod.Method;
var host = Encoding.ASCII.GetString(startLine.Slice(targetPath.Offset, targetPath.Length));
if (versionAndMethod.Method == HttpMethod.Connect)
{
ProxyHost = HostString.FromUriComponent(host);
}
else if (Uri.TryCreate(host, UriKind.Absolute, out var uri))
{
ProxyHost = HostString.FromUriComponent(uri);
}
}
void IHttpHeadersHandler.OnHeader(ReadOnlySpan<byte> name, ReadOnlySpan<byte> value)
{
}
void IHttpHeadersHandler.OnHeadersComplete(bool endStream)
{
}
void IHttpHeadersHandler.OnStaticIndexedHeader(int index)
{
}
void IHttpHeadersHandler.OnStaticIndexedHeader(int index, ReadOnlySpan<byte> value)
{
}
}
}
Fiddler只監聽了一個埠，要同時支援非加密和加密兩種流量，如果不呼叫呼叫ListenOptions.UseHttps()，我們的程式就不支援https的分析；如果直接呼叫ListenOptions.UseHttps()，會讓我們的程式不支援非加密的http的分析，這就要求我們有條件的根據使用者端發來的流量分析是否需要開啟。
我已經在KestrelFramework內建了TlsDetection中介軟體，這個中介軟體可以根據使用者端的實際流量型別來選擇是否使用tls。在Fiddler中，我們還需要根據使用者端的tls握手中的sni使用ca證書來動態生成伺服器證書用於tls加密傳輸。
/// <summary>
/// 證書服務
/// </summary>
sealed class CertService
{
private const string CACERT_PATH = "cacert";
private readonly IMemoryCache serverCertCache;
private readonly IEnumerable<ICaCertInstaller> certInstallers;
private readonly ILogger<CertService> logger;
private X509Certificate2? caCert;
/// <summary>
/// 獲取證書檔案路徑
/// </summary>
public string CaCerFilePath { get; } = OperatingSystem.IsLinux() ? $"{CACERT_PATH}/fiddler.crt" : $"{CACERT_PATH}/fiddler.cer";
/// <summary>
/// 獲取私鑰檔案路徑
/// </summary>
public string CaKeyFilePath { get; } = $"{CACERT_PATH}/fiddler.key";
/// <summary>
/// 證書服務
/// </summary>
/// <param name="serverCertCache"></param>
/// <param name="certInstallers"></param>
/// <param name="logger"></param>
public CertService(
IMemoryCache serverCertCache,
IEnumerable<ICaCertInstaller> certInstallers,
ILogger<CertService> logger)
{
this.serverCertCache = serverCertCache;
this.certInstallers = certInstallers;
this.logger = logger;
Directory.CreateDirectory(CACERT_PATH);
}
/// <summary>
/// 生成CA證書
/// </summary>
public bool CreateCaCertIfNotExists()
{
if (File.Exists(this.CaCerFilePath) && File.Exists(this.CaKeyFilePath))
{
return false;
}
File.Delete(this.CaCerFilePath);
File.Delete(this.CaKeyFilePath);
var notBefore = DateTimeOffset.Now.AddDays(-1);
var notAfter = DateTimeOffset.Now.AddYears(10);
var subjectName = new X500DistinguishedName($"CN={nameof(Fiddler)}");
this.caCert = CertGenerator.CreateCACertificate(subjectName, notBefore, notAfter);
var privateKeyPem = this.caCert.GetRSAPrivateKey()?.ExportRSAPrivateKeyPem();
File.WriteAllText(this.CaKeyFilePath, new string(privateKeyPem), Encoding.ASCII);
var certPem = this.caCert.ExportCertificatePem();
File.WriteAllText(this.CaCerFilePath, new string(certPem), Encoding.ASCII);
return true;
}
/// <summary>
/// 安裝和信任CA證書
/// </summary>
public void InstallAndTrustCaCert()
{
var installer = this.certInstallers.FirstOrDefault(item => item.IsSupported());
if (installer != null)
{
installer.Install(this.CaCerFilePath);
}
else
{
this.logger.LogWarning($"請根據你的系統平臺手動安裝和信任CA證書{this.CaCerFilePath}");
}
}
/// <summary>
/// 獲取頒發給指定域名的證書
/// </summary>
/// <param name="domain"></param>
/// <returns></returns>
public X509Certificate2 GetOrCreateServerCert(string? domain)
{
if (this.caCert == null)
{
using var rsa = RSA.Create();
rsa.ImportFromPem(File.ReadAllText(this.CaKeyFilePath));
this.caCert = new X509Certificate2(this.CaCerFilePath).CopyWithPrivateKey(rsa);
}
var key = $"{nameof(CertService)}:{domain}";
var endCert = this.serverCertCache.GetOrCreate(key, GetOrCreateCert);
return endCert!;
// 生成域名的1年證書
X509Certificate2 GetOrCreateCert(ICacheEntry entry)
{
var notBefore = DateTimeOffset.Now.AddDays(-1);
var notAfter = DateTimeOffset.Now.AddYears(1);
entry.SetAbsoluteExpiration(notAfter);
var extraDomains = GetExtraDomains();
var subjectName = new X500DistinguishedName($"CN={domain}");
var endCert = CertGenerator.CreateEndCertificate(this.caCert, subjectName, extraDomains, notBefore, notAfter);
// 重新初始化證書，以相容win平臺不能使用記憶體證書
return new X509Certificate2(endCert.Export(X509ContentType.Pfx));
}
}
/// <summary>
/// 獲取域名
/// </summary>
/// <param name="domain"></param>
/// <returns></returns>
private static IEnumerable<string> GetExtraDomains()
{
yield return Environment.MachineName;
yield return IPAddress.Loopback.ToString();
yield return IPAddress.IPv6Loopback.ToString();
}
}
經過KestrelProxyMiddleware後的流量，在tls解密(如果可能)之後，一般情況下都是http流量了，但如果你在qq設定代理到我們這個偽Fildder之後，會發現部分流量流量不是http流量，原因是http隧道也是一個通用傳輸層，可以傳輸任意tcp或tcp之上的流量。所以我們需要新的中介軟體來檢測當前流量，如果不是http流量就回退到隧道代理的流程，即我們不跟蹤不分析這部分非http流量。
/// <summary>
/// 流量偵測器
/// </summary>
private static class FlowDetector
{
private static readonly byte[] crlf = Encoding.ASCII.GetBytes("\r\n");
private static readonly byte[] http10 = Encoding.ASCII.GetBytes(" HTTP/1.0");
private static readonly byte[] http11 = Encoding.ASCII.GetBytes(" HTTP/1.1");
private static readonly byte[] http20 = Encoding.ASCII.GetBytes(" HTTP/2.0");
/// <summary>
/// 傳輸內容是否為http
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public static async ValueTask<bool> IsHttpAsync(ConnectionContext context)
{
var input = context.Transport.Input;
var result = await input.ReadAtLeastAsync(1);
var isHttp = IsHttp(result);
input.AdvanceTo(result.Buffer.Start);
return isHttp;
}
private static bool IsHttp(ReadResult result)
{
var reader = new SequenceReader<byte>(result.Buffer);
if (reader.TryReadToAny(out ReadOnlySpan<byte> line, crlf))
{
return line.EndsWith(http11) || line.EndsWith(http20) || line.EndsWith(http10);
}
return false;
}
}
/// <summary>
/// 隧道傳輸中介軟體
/// </summary>
sealed class KestrelTunnelMiddleware : IKestrelMiddleware
{
private readonly ILogger<KestrelTunnelMiddleware> logger;
/// <summary>
/// 隧道傳輸中介軟體
/// </summary>
/// <param name="logger"></param>
public KestrelTunnelMiddleware(ILogger<KestrelTunnelMiddleware> logger)
{
this.logger = logger;
}
/// <summary>
/// 執行中間你件
/// </summary>
/// <param name="next"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task InvokeAsync(ConnectionDelegate next, ConnectionContext context)
{
var feature = context.Features.Get<IProxyFeature>();
if (feature == null || feature.ProxyProtocol == ProxyProtocol.None)
{
this.logger.LogInformation($"偵測到http直接請求");
await next(context);
}
else if (feature.ProxyProtocol == ProxyProtocol.HttpProxy)
{
this.logger.LogInformation($"偵測到普通http代理流量");
await next(context);
}
else if (await FlowDetector.IsHttpAsync(context))
{
this.logger.LogInformation($"偵測到隧道傳輸http流量");
await next(context);
}
else
{
this.logger.LogInformation($"跳過隧道傳輸非http流量{feature.ProxyHost}的攔截");
await TunnelAsync(context, feature);
}
}
/// <summary>
/// 隧道傳輸其它協定的資料
/// </summary>
/// <param name="context"></param>
/// <param name="feature"></param>
/// <returns></returns>
private async ValueTask TunnelAsync(ConnectionContext context, IProxyFeature feature)
{
var port = feature.ProxyHost.Port;
if (port == null)
{
return;
}
try
{
var host = feature.ProxyHost.Host;
using var socket = new Socket(SocketType.Stream, ProtocolType.Tcp);
await socket.ConnectAsync(host, port.Value, context.ConnectionClosed);
Stream stream = new NetworkStream(socket, ownsSocket: false);
// 如果有tls中介軟體，則反回來加密隧道
if (context.Features.Get<ITlsConnectionFeature>() != null)
{
var sslStream = new SslStream(stream, leaveInnerStreamOpen: true);
await sslStream.AuthenticateAsClientAsync(feature.ProxyHost.Host);
stream = sslStream;
}
var task1 = stream.CopyToAsync(context.Transport.Output);
var task2 = context.Transport.Input.CopyToAsync(stream);
await Task.WhenAny(task1, task2);
}
catch (Exception ex)
{
this.logger.LogError(ex, $"連線到{feature.ProxyHost}異常");
}
}
}
這部分屬於asp.netcore應用層內容，關鍵點是製作可多次讀取的http請求body流和http響應body流，因為每個分析器範例都可以會重頭讀取一次請求內容和響應內容。
為了方便各種分析器的獨立實現，我們定義http分析器的介面
/// <summary>
/// http分析器
/// 支援多個範例
/// </summary>
public interface IHttpAnalyzer
{
/// <summary>
/// 分析http
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
ValueTask AnalyzeAsync(HttpContext context);
}
這是輸到紀錄檔的http分析器
public class LoggingHttpAnalyzer : IHttpAnalyzer
{
private readonly ILogger<LoggingHttpAnalyzer> logger;
public LoggingHttpAnalyzer(ILogger<LoggingHttpAnalyzer> logger)
{
this.logger = logger;
}
public async ValueTask AnalyzeAsync(HttpContext context)
{
var builder = new StringBuilder();
var writer = new StringWriter(builder);
writer.WriteLine("[REQUEST]");
await context.SerializeRequestAsync(writer);
writer.WriteLine("[RESPONSE]");
await context.SerializeResponseAsync(writer);
this.logger.LogInformation(builder.ToString());
}
}
我們把請求body流和響應body流儲存到臨時檔案，在所有分析器工作之後再刪除。
/// <summary>
/// http分析中介軟體
/// </summary>
sealed class HttpAnalyzeMiddleware
{
private readonly RequestDelegate next;
private readonly IEnumerable<IHttpAnalyzer> analyzers;
/// <summary>
/// http分析中介軟體
/// </summary>
/// <param name="next"></param>
/// <param name="analyzers"></param>
public HttpAnalyzeMiddleware(
RequestDelegate next,
IEnumerable<IHttpAnalyzer> analyzers)
{
this.next = next;
this.analyzers = analyzers;
}
/// <summary>
/// 分析代理的http流量
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public async Task InvokeAsync(HttpContext context)
{
var feature = context.Features.Get<IProxyFeature>();
if (feature == null || feature.ProxyProtocol == ProxyProtocol.None)
{
await next(context);
return;
}
context.Request.EnableBuffering();
var oldBody = context.Response.Body;
using var response = new FileResponse();
try
{
// 替換response的body
context.Response.Body = response.Body;
// 請求下箇中介軟體
await next(context);
// 處理分析
await this.AnalyzeAsync(context);
}
finally
{
response.Body.Position = 0L;
await response.Body.CopyToAsync(oldBody);
context.Response.Body = oldBody;
}
}
private async ValueTask AnalyzeAsync(HttpContext context)
{
foreach (var item in this.analyzers)
{
context.Request.Body.Position = 0L;
context.Response.Body.Position = 0L;
await item.AnalyzeAsync(context);
}
}
private class FileResponse : IDisposable
{
private readonly string filePath = Path.GetTempFileName();
public Stream Body { get; }
public FileResponse()
{
this.Body = new FileStream(filePath, FileMode.Open, FileAccess.ReadWrite);
}
public void Dispose()
{
this.Body.Dispose();
File.Delete(filePath);
}
}
}
我們需要把請求轉發到真實的目標伺服器，這時我們的應用程式是一個http使用者端角色，這個過程與nginx的反向代理是一致的。具體的實現上，我們直接使用yarp庫來完成即可。
/// <summary>
/// http代理執行中介軟體
/// </summary>
sealed class HttpForwardMiddleware
{
private readonly RequestDelegate next;
private readonly IHttpForwarder httpForwarder;
private readonly HttpMessageInvoker httpClient = new(CreateSocketsHttpHandler());
/// <summary>
/// http代理執行中介軟體
/// </summary>
/// <param name="next"></param>
/// <param name="httpForwarder"></param>
public HttpForwardMiddleware(
RequestDelegate next,
IHttpForwarder httpForwarder)
{
this.next = next;
this.httpForwarder = httpForwarder;
}
/// <summary>
/// 轉發http流量
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public async Task InvokeAsync(HttpContext context)
{
var feature = context.Features.Get<IProxyFeature>();
if (feature == null || feature.ProxyProtocol == ProxyProtocol.None)
{
await next(context);
}
else
{
var scheme = context.Request.Scheme;
var destinationPrefix = $"{scheme}://{feature.ProxyHost}";
await httpForwarder.SendAsync(context, destinationPrefix, httpClient, ForwarderRequestConfig.Empty, HttpTransformer.Empty);
}
}
private static SocketsHttpHandler CreateSocketsHttpHandler()
{
return new SocketsHttpHandler
{
Proxy = null,
UseProxy = false,
UseCookies = false,
AllowAutoRedirect = false,
AutomaticDecompression = DecompressionMethods.None,
};
}
}
這裡要特別注意順序，傳輸層套娃。
/// <summary>
/// ListenOptions擴充套件
/// </summary>
public static partial class ListenOptionsExtensions
{
/// <summary>
/// 使用Fiddler的kestrel中介軟體
/// </summary>
/// <param name="listen"></param>
public static ListenOptions UseFiddler(this ListenOptions listen)
{
// 代理協定中介軟體
listen.Use<KestrelProxyMiddleware>();
// tls偵測中介軟體
listen.UseTlsDetection(tls =>
{
var certService = listen.ApplicationServices.GetRequiredService<CertService>();
certService.CreateCaCertIfNotExists();
certService.InstallAndTrustCaCert();
tls.ServerCertificateSelector = (context, domain) => certService.GetOrCreateServerCert(domain);
});
// 隧道代理處理中介軟體
listen.Use<KestrelTunnelMiddleware>();
return listen;
}
}
public static class ApplicationBuilderExtensions
{
/// <summary>
/// 使用Fiddler的http中介軟體
/// </summary>
/// <param name="app"></param>
public static void UseFiddler(this IApplicationBuilder app)
{
app.UseMiddleware<HttpAnalyzeMiddleware>();
app.UseMiddleware<HttpForwardMiddleware>();
}
}
我們可以在傳統的MVC裡建立偽fiddler的首頁、下載證書等http互動頁面。
public static void Main(string[] args)
{
var builder = WebApplication.CreateBuilder(args);
builder.Services
.AddFiddler()
.AddControllers();
builder.WebHost.ConfigureKestrel((context, kestrel) =>
{
var section = context.Configuration.GetSection("Kestrel");
kestrel.Configure(section).Endpoint("Fiddler", endpoint => endpoint.ListenOptions.UseFiddler());
});
var app = builder.Build();
app.UseRouting();
app.UseFiddler();
app.MapControllerRoute(
name: "default",
pattern: "{controller=Home}/{action=Index}/{id?}");
app.Run();
}
如果讓您來開發個偽Fiddler，除了本文的方法，您會使用什麼方式來開發呢？