Skip to content

Commit

Permalink
proxy: match NO_PROXY formats to libcurl behaviour
Browse files Browse the repository at this point in the history 
We aim to be compatible with the behaviour of Git as much as possible
when it comes to network settings. This enables users to setup Git proxy
settings and get the same setup "for free" with GCM.

Git uses libcurl to provide it's HTTP interactions. The NO_PROXY setting
is used by libcurl to disable proxy settings for specific hosts.

We previously attempted to plumb the value of NO_PROXY through to the
.NET WebProxy class' list of "bypassed addresses" (the set of hosts that
should not be proxied). However, the .NET class expects a set of
_regular expressions_ which is unlike libcurl!

As a result, libcurl permitted values for NO_PROXY were throwing errors
inside of GCM since they are not valid regexs.

In this commit we perform a transformation of the NO_PROXY list and
construct a set of regular expressions that match addresses in the same
way as libcurl does.

The transformation is as follows:

1. strip any leading periods '.' or wildcards '*.'
2. escape the remaining input to match literally (e.g.: '.' becomes '\.')
3. prepend a group that matches either a period '.' or a URI scheme
   delimiter '://' - this prevents partial domain matching
4. append a end-of-string symbol '$' to ensure we only match to the
   specified TLD and port

See the libcurl documentation on NO_PROXY behaviour:
https://curl.se/libcurl/c/CURLOPT_NOPROXY.html
  • Loading branch information
@mjcheetham
mjcheetham committed Oct 20, 2021
1 parent d872294 commit f466150
Show file tree
Hide file tree
Showing 5 changed files with 221 additions and 73 deletions.
24 changes: 22 additions & 2 deletions docs/netconfig.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,28 @@ addresses. GCM Core supports the cURL environment variable `NO_PROXY` for this
scenariom, as does Git itself.

The `NO_PROXY` environment variable should contain a comma (`,`) or space (` `)
separated list of regular expressions to match hosts that should not be proxied
(should connect directly).
separated list of host names that should not be proxied (should connect
directly).

GCM Core attempts to match [libcurl's behaviour](https://curl.se/libcurl/c/CURLOPT_NOPROXY.html),
which is briefly summarized here:

- a value of `*` disables proxying for all hosts;
- other wildcard use is **not** supported;
- each name in the list is matched as a domain which contains the hostname,
or the hostname itself
- a leading period/dot `.` matches against the provided hostname

For example, setting `NO_PROXY` to `example.com` results in the following:

Hostname|Matches?
-|-
`example.com`|:white_check_mark:
`example.com:80`|:white_check_mark:
`www.example.com`|:white_check_mark:
`notanexample.com`|:x:
`www.notanexample.com`|:x:
`example.com.othertld`|:x:

**Example:**

Expand Down
33 changes: 31 additions & 2 deletions src/shared/Microsoft.Git.CredentialManager.Tests/HttpClientFactoryTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,13 @@ public void HttpClientFactory_TryCreateProxy_ProxyWithBypass_ReturnsTrueOutProxy
const string repoPath = "/tmp/repos/foo";
const string repoRemote = "https://remote.example.com/foo.git";

var bypassList = new List<string> {"https://contoso.com", ".*fabrikam\\.com"};
var noProxyRaw = "contoso.com,fabrikam.com";
var repoRemoteUri = new Uri(repoRemote);
var proxyConfig = new ProxyConfiguration(
new Uri(proxyUrl),
userName: null,
password: null,
bypassHosts: bypassList);
noProxyRaw: noProxyRaw);

var settings = new TestSettings
{
Expand All @@ -117,6 +117,35 @@ public void HttpClientFactory_TryCreateProxy_ProxyWithBypass_ReturnsTrueOutProxy
Assert.False(proxy.IsBypassed(repoRemoteUri));
}

[Fact]
public void HttpClientFactory_TryCreateProxy_ProxyWithWildcardBypass_ReturnsFalse()
{
const string proxyUrl = "https://proxy.example.com/git";
const string repoPath = "/tmp/repos/foo";
const string repoRemote = "https://remote.example.com/foo.git";

var noProxyRaw = "*";
var repoRemoteUri = new Uri(repoRemote);
var proxyConfig = new ProxyConfiguration(
new Uri(proxyUrl),
userName: null,
password: null,
noProxyRaw: noProxyRaw);

var settings = new TestSettings
{
RemoteUri = repoRemoteUri,
RepositoryPath = repoPath,
ProxyConfiguration = proxyConfig
};
var httpFactory = new HttpClientFactory(Mock.Of<IFileSystem>(), Mock.Of<ITrace>(), settings, Mock.Of<IStandardStreams>());

bool result = httpFactory.TryCreateProxy(out IWebProxy proxy);

Assert.False(result);
Assert.Null(proxy);
}

[Fact]
public void HttpClientFactory_TryCreateProxy_ProxyWithCredentials_ReturnsTrueOutProxyWithUrlConfiguredCredentials()
{
Expand Down
128 changes: 71 additions & 57 deletions src/shared/Microsoft.Git.CredentialManager.Tests/SettingsTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using Microsoft.Git.CredentialManager.Tests.Objects;
using Xunit;

Expand Down Expand Up @@ -427,6 +428,55 @@ public void Settings_IsWindowsIntegratedAuthenticationEnabled_ConfigNonBooleanyV
Assert.True(settings.IsWindowsIntegratedAuthenticationEnabled);
}

[Theory]
[InlineData("", new string[0])]
[InlineData(" ", new string[0])]
[InlineData(",", new string[0])]
[InlineData("example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
[InlineData("example.com:8080", new[] { @"(\.|\:\/\/)example\.com:8080$" })]
[InlineData("example.com,", new[] { @"(\.|\:\/\/)example\.com$" })]
[InlineData(",example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
[InlineData(",example.com,", new[] { @"(\.|\:\/\/)example\.com$" })]
[InlineData(".example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
[InlineData("..example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
[InlineData("*.example.com", new[] { @"(\.|\:\/\/)example\.com$" })]
[InlineData("my.example.com", new[] { @"(\.|\:\/\/)my\.example\.com$" })]
[InlineData("example.com,contoso.com,fabrikam.com", new[]
{
@"(\.|\:\/\/)example\.com$",
@"(\.|\:\/\/)contoso\.com$",
@"(\.|\:\/\/)fabrikam\.com$"
})]
public void Settings_ProxyConfiguration_ConvertToBypassRegexArray(string input, string[] expected)
{
string[] actual = ProxyConfiguration.ConvertToBypassRegexArray(input).ToArray();
Assert.Equal(expected, actual);
}

[Theory]
[InlineData("example.com", "http://example.com", true)]
[InlineData("example.com", "https://example.com", true)]
[InlineData("example.com", "https://www.example.com", true)]
[InlineData("example.com", "http://www.example.com:80", true)]
[InlineData("example.com", "https://www.example.com:443", true)]
[InlineData("example.com", "https://www.example.com:8080", false)]
[InlineData("example.com", "http://notanexample.com", false)]
[InlineData("example.com", "https://notanexample.com", false)]
[InlineData("example.com", "https://www.notanexample.com", false)]
[InlineData("example.com", "https://example.com.otherltd", false)]
public void Settings_ProxyConfiguration_ConvertToBypassRegexArray_WebProxyBypass(string noProxy, string address, bool expected)
{
var bypassList = ProxyConfiguration.ConvertToBypassRegexArray(noProxy).ToArray();
var webProxy = new WebProxy("https://localhost:8080/proxy")
{
BypassList = bypassList
};

bool actual = webProxy.IsBypassed(new Uri(address));

Assert.Equal(expected, actual);
}

[Fact]
public void Settings_ProxyConfiguration_Unset_ReturnsNull()
{
Expand Down Expand Up @@ -458,11 +508,11 @@ public void Settings_ProxyConfiguration_GcmHttpConfig_ReturnsValue()
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
var expectedNoProxy = "contoso.com,fabrikam.com";

var envars = new TestEnvironment
{
Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)}
Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy}
};
var git = new TestGit();
git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};
Expand All @@ -478,7 +528,7 @@ public void Settings_ProxyConfiguration_GcmHttpConfig_ReturnsValue()
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
Assert.True(actualConfig.IsDeprecatedSource);
}

Expand All @@ -494,11 +544,11 @@ public void Settings_ProxyConfiguration_GcmHttpsConfig_ReturnsValue()
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
var expectedNoProxy = "contoso.com,fabrikam.com";

var envars = new TestEnvironment
{
Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)}
Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy}
};
var git = new TestGit();
git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};
Expand All @@ -514,7 +564,7 @@ public void Settings_ProxyConfiguration_GcmHttpsConfig_ReturnsValue()
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
Assert.True(actualConfig.IsDeprecatedSource);
}

Expand All @@ -530,11 +580,11 @@ public void Settings_ProxyConfiguration_GitHttpConfig_ReturnsValue()
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
var expectedNoProxy = "contoso.com,fabrikam.com";

var envars = new TestEnvironment
{
Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)}
Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy}
};
var git = new TestGit();
git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};
Expand All @@ -550,7 +600,7 @@ public void Settings_ProxyConfiguration_GitHttpConfig_ReturnsValue()
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
Assert.False(actualConfig.IsDeprecatedSource);
}

Expand Down Expand Up @@ -579,42 +629,6 @@ public void Settings_ProxyConfiguration_GitHttpConfig_EmptyScopedUriUnscoped_Ret
Assert.Null(actualConfig);
}

[Fact]
public void Settings_ProxyConfiguration_NoProxyMixedSplitChar_ReturnsValue()
{
const string remoteUrl = "http://example.com/foo.git";
const string section = Constants.GitConfiguration.Http.SectionName;
const string property = Constants.GitConfiguration.Http.Proxy;
var remoteUri = new Uri(remoteUrl);

const string expectedUserName = "john.doe";
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"contoso.com", "fabrikam.com", "example.com"};

var envars = new TestEnvironment
{
Variables = {[Constants.EnvironmentVariables.CurlNoProxy] = "contoso.com, fabrikam.com example.com,"}
};
var git = new TestGit();
git.Configuration.Global[$"{section}.{property}"] = new[] {settingValue.ToString()};

var settings = new Settings(envars, git)
{
RemoteUri = remoteUri
};

ProxyConfiguration actualConfig = settings.GetProxyConfiguration();

Assert.NotNull(actualConfig);
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.False(actualConfig.IsDeprecatedSource);
}

[Fact]
public void Settings_ProxyConfiguration_CurlHttpEnvar_ReturnsValue()
{
Expand All @@ -625,14 +639,14 @@ public void Settings_ProxyConfiguration_CurlHttpEnvar_ReturnsValue()
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
var expectedNoProxy = "contoso.com,fabrikam.com";

var envars = new TestEnvironment
{
Variables =
{
[Constants.EnvironmentVariables.CurlHttpProxy] = settingValue.ToString(),
[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
}
};
var git = new TestGit();
Expand All @@ -648,7 +662,7 @@ public void Settings_ProxyConfiguration_CurlHttpEnvar_ReturnsValue()
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
Assert.False(actualConfig.IsDeprecatedSource);
}

Expand All @@ -662,14 +676,14 @@ public void Settings_ProxyConfiguration_CurlHttpsEnvar_ReturnsValue()
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
var expectedNoProxy = "contoso.com,fabrikam.com";

var envars = new TestEnvironment
{
Variables =
{
[Constants.EnvironmentVariables.CurlHttpsProxy] = settingValue.ToString(),
[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
}
};
var git = new TestGit();
Expand All @@ -685,7 +699,7 @@ public void Settings_ProxyConfiguration_CurlHttpsEnvar_ReturnsValue()
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
Assert.False(actualConfig.IsDeprecatedSource);
}

Expand All @@ -699,14 +713,14 @@ public void Settings_TryGetProxy_CurlAllEnvar_ReturnsValue()
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"contoso.com", "fabrikam.com"};
var expectedNoProxy = "contoso.com,fabrikam.com";

var envars = new TestEnvironment
{
Variables =
{
[Constants.EnvironmentVariables.CurlAllProxy] = settingValue.ToString(),
[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
}
};
var git = new TestGit();
Expand All @@ -722,7 +736,7 @@ public void Settings_TryGetProxy_CurlAllEnvar_ReturnsValue()
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
Assert.False(actualConfig.IsDeprecatedSource);
}

Expand All @@ -736,14 +750,14 @@ public void Settings_ProxyConfiguration_LegacyGcmEnvar_ReturnsValue()
const string expectedPassword = "letmein123";
var expectedAddress = new Uri("http://proxy.example.com");
var settingValue = new Uri("http://john.doe:[email protected]");
var bypassList = new List<string> {"https://contoso.com", ".*fabrikam\\.com"};
var expectedNoProxy = "contoso.com,fabrikam.com";

var envars = new TestEnvironment
{
Variables =
{
[Constants.EnvironmentVariables.GcmHttpProxy] = settingValue.ToString(),
[Constants.EnvironmentVariables.CurlNoProxy] = string.Join(',', bypassList)
[Constants.EnvironmentVariables.CurlNoProxy] = expectedNoProxy
}
};
var git = new TestGit();
Expand All @@ -759,7 +773,7 @@ public void Settings_ProxyConfiguration_LegacyGcmEnvar_ReturnsValue()
Assert.Equal(expectedAddress, actualConfig.Address);
Assert.Equal(expectedUserName, actualConfig.UserName);
Assert.Equal(expectedPassword, actualConfig.Password);
Assert.Equal(bypassList, actualConfig.BypassHosts);
Assert.Equal(expectedNoProxy, actualConfig.NoProxyRaw);
Assert.True(actualConfig.IsDeprecatedSource);
}

Expand Down
Loading

0 comments on commit f466150

Please sign in to comment.