PATCH: Add $fqdn stream and http variable.

[David Coles]

# HG changeset patch
# User David Coles <***@gmail.com>
# Date 1543000128 0
# Fri Nov 23 19:08:48 2018 +0000
# Node ID 8e014e3bdfbda3c7a22c7980c163bcea1c4474b3
# Parent be5cb9c67c05ccaf22dab7abba78aa4c1545a8ee
Add $fqdn stream and http variable.

This exposes the system's fully-qualified domain name as reported by calling
getaddrbyname on the hostname. Typically $hostname will only contain the first
segment of the host's domainname.

diff -r be5cb9c67c05 -r 8e014e3bdfbd src/core/ngx_config.h
--- a/src/core/ngx_config.h Wed Nov 21 20:23:16 2018 +0300
+++ b/src/core/ngx_config.h Fri Nov 23 19:08:48 2018 +0000
@@ -124,6 +124,11 @@
#define NGX_MAXHOSTNAMELEN 256
#endif

+#ifdef MAXFQDNLEN
+#define NGX_MAXFQDNLEN MAXFQDNLEN
+#else
+#define NGX_MAXFQDNLEN 256
+#endif

#define NGX_MAX_UINT32_VALUE (uint32_t) 0xffffffff
#define NGX_MAX_INT32_VALUE (uint32_t) 0x7fffffff
diff -r be5cb9c67c05 -r 8e014e3bdfbd src/core/ngx_cycle.c
--- a/src/core/ngx_cycle.c Wed Nov 21 20:23:16 2018 +0300
+++ b/src/core/ngx_cycle.c Fri Nov 23 19:08:48 2018 +0000
@@ -9,6 +9,8 @@
#include <ngx_core.h>
#include <ngx_event.h>

+#include <netdb.h>
+

static void ngx_destroy_cycle_pools(ngx_conf_t *conf);
static ngx_int_t ngx_init_zone_pool(ngx_cycle_t *cycle,
@@ -53,6 +55,7 @@
ngx_core_conf_t *ccf, *old_ccf;
ngx_core_module_t *module;
char hostname[NGX_MAXHOSTNAMELEN];
+ struct hostent *hostent;

ngx_timezone_update();

@@ -213,6 +216,23 @@
ngx_strlow(cycle->hostname.data, (u_char *) hostname, cycle->hostname.len);


+ hostent = gethostbyname(hostname);
+ if (hostent == NULL) {
+ ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "gethostbyname() failed");
+ ngx_destroy_pool(pool);
+ return NULL;
+ }
+
+ cycle->fqdn.len = ngx_strlen(hostent->h_name);
+
+ cycle->fqdn.data = ngx_pnalloc(pool, cycle->fqdn.len);
+ if (cycle->fqdn.data == NULL) {
+ ngx_destroy_pool(pool);
+ return NULL;
+ }
+ ngx_strlow(cycle->fqdn.data, (u_char *) hostent->h_name, cycle->fqdn.len);
+
+
if (ngx_cycle_modules(cycle) != NGX_OK) {
ngx_destroy_pool(pool);
return NULL;
diff -r be5cb9c67c05 -r 8e014e3bdfbd src/core/ngx_cycle.h
--- a/src/core/ngx_cycle.h Wed Nov 21 20:23:16 2018 +0300
+++ b/src/core/ngx_cycle.h Fri Nov 23 19:08:48 2018 +0000
@@ -81,6 +81,7 @@
ngx_str_t prefix;
ngx_str_t lock_file;
ngx_str_t hostname;
+ ngx_str_t fqdn;
};


diff -r be5cb9c67c05 -r 8e014e3bdfbd src/http/ngx_http_variables.c
--- a/src/http/ngx_http_variables.c Wed Nov 21 20:23:16 2018 +0300
+++ b/src/http/ngx_http_variables.c Fri Nov 23 19:08:48 2018 +0000
@@ -134,6 +134,8 @@
ngx_http_variable_value_t *v, uintptr_t data);
static ngx_int_t ngx_http_variable_hostname(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_http_variable_fqdn(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data);
static ngx_int_t ngx_http_variable_pid(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
static ngx_int_t ngx_http_variable_msec(ngx_http_request_t *r,
@@ -338,6 +340,9 @@
{ ngx_string("hostname"), NULL, ngx_http_variable_hostname,
0, 0, 0 },

+ { ngx_string("fqdn"), NULL, ngx_http_variable_fqdn,
+ 0, 0, 0 },
+
{ ngx_string("pid"), NULL, ngx_http_variable_pid,
0, 0, 0 },

@@ -2256,6 +2261,20 @@


static ngx_int_t
+ngx_http_variable_fqdn(ngx_http_request_t *r,
+ ngx_http_variable_value_t *v, uintptr_t data)
+{
+ v->len = ngx_cycle->fqdn.len;
+ v->valid = 1;
+ v->no_cacheable = 0;
+ v->not_found = 0;
+ v->data = ngx_cycle->fqdn.data;
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
ngx_http_variable_pid(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data)
{
diff -r be5cb9c67c05 -r 8e014e3bdfbd src/stream/ngx_stream_variables.c
--- a/src/stream/ngx_stream_variables.c Wed Nov 21 20:23:16 2018 +0300
+++ b/src/stream/ngx_stream_variables.c Fri Nov 23 19:08:48 2018 +0000
@@ -40,6 +40,8 @@
ngx_stream_variable_value_t *v, uintptr_t data);
static ngx_int_t ngx_stream_variable_hostname(ngx_stream_session_t *s,
ngx_stream_variable_value_t *v, uintptr_t data);
+static ngx_int_t ngx_stream_variable_fqdn(ngx_stream_session_t *s,
+ ngx_stream_variable_value_t *v, uintptr_t data);
static ngx_int_t ngx_stream_variable_pid(ngx_stream_session_t *s,
ngx_stream_variable_value_t *v, uintptr_t data);
static ngx_int_t ngx_stream_variable_msec(ngx_stream_session_t *s,
@@ -96,6 +98,9 @@
{ ngx_string("hostname"), NULL, ngx_stream_variable_hostname,
0, 0, 0 },

+ { ngx_string("fqdn"), NULL, ngx_stream_variable_fqdn,
+ 0, 0, 0 },
+
{ ngx_string("pid"), NULL, ngx_stream_variable_pid,
0, 0, 0 },

@@ -778,6 +783,20 @@


static ngx_int_t
+ngx_stream_variable_fqdn(ngx_stream_session_t *s,
+ ngx_stream_variable_value_t *v, uintptr_t data)
+{
+ v->len = ngx_cycle->fqdn.len;
+ v->valid = 1;
+ v->no_cacheable = 0;
+ v->not_found = 0;
+ v->data = ngx_cycle->fqdn.data;
+
+ return NGX_OK;
+}
+
+
+static ngx_int_t
ngx_stream_variable_pid(ngx_stream_session_t *s,
ngx_stream_variable_value_t *v, uintptr_t data)
{

[Maxim Dounin]

Hello!

Post by David Coles
# HG changeset patch
# Date 1543000128 0
# Fri Nov 23 19:08:48 2018 +0000
# Node ID 8e014e3bdfbda3c7a22c7980c163bcea1c4474b3
# Parent be5cb9c67c05ccaf22dab7abba78aa4c1545a8ee
Add $fqdn stream and http variable.
This exposes the system's fully-qualified domain name as reported by calling
getaddrbyname on the hostname. Typically $hostname will only contain the first
segment of the host's domainname.

[...]

No, thank you. This change makes nginx startup unconditionally
dependant on the DNS availability. This is not something we are
going to accept.

Also, the $hostname variable is expected to be FQDN on systems
participating in the DNS. If it's not FQDN in your case, you may
want to consider reconfiguring your system. See
http://mailman.nginx.org/pipermail/nginx/2017-February/052885.html
for a previous discussion on the topic.

[David Coles]

Hi Maxim. Thanks for your comments.

Post by Maxim Dounin
No, thank you. This change makes nginx startup unconditionally
dependant on the DNS availability. This is not something we are
going to accept.

That's reasonable. I based this behaviour off `hostname -f` which also
uses `gethostbyname` for resolving the FQDN.
Note, on most systems this doesn't trigger an actual DNS lookup
provided that hosts canonical name (usually the FQDN) is configured in
`/etc/hosts` (See paragraph 5 of THE FQDN
<https://manpages.debian.org/testing/hostname/hostname.1.en.html>).

On some systems `gethostent` can be used to explicitly walk
`/etc/hosts`, but behaviour is so inconsistent I would avoid it.

How would you feel about trying to resolve the FQDN at nginx startup
and, if unavailable falling back to `$hostname`? This matches the
behaviour of other languages (e.g. Python's `socket.getfqdn()`
<https://docs.python.org/3/library/socket.html#socket.getfqdn>). I
could also make this a runtime evaluation, but it would be nice to
only evaluate once as it would often be used in hot paths like logging
and tracing.

Post by Maxim Dounin
Also, the $hostname variable is expected to be FQDN on systems
participating in the DNS. If it's not FQDN in your case, you may
want to consider reconfiguring your system. See
http://mailman.nginx.org/pipermail/nginx/2017-February/052885.html
for a previous discussion on the topic.

Thank you for the pointer to the previous discussion. Doing some
investigation I've found several contrary recommendations:

https://www.gnu.org/software/libc/manual/html_node/Host-Identification.html

: Function: int gethostname (char *name, size_t size)
...
: If the system participates in the DNS, this is the FQDN (see
: above).

https://manpages.debian.org/testing/hostname/hostname.1.en.html

: The recommended method of setting the FQDN is to make the hostname be an alias
: for the fully qualified name using /etc/hosts, DNS, or NIS. For
example, if the
: hostname was "ursula", one might have a line in /etc/hosts which reads
:
: 127.0.1.1 ursula.example.com ursula

The Linux programming interface. Kerrisk, M. (2010). §12.2 (p. 299)
<https://books.google.com/books?id=Ps2SH727eCIC&lpg=PR7&ots=kMFex6wMt7&lr&pg=PA229#v=onepage&q&f=false>

: Often, this name is something like the hostname prefix from the system’s DNS
: domain name.

Advanced programming in the UNIX environment. Stevens, W. R., & Rago,
S. A. (2013). §6.9 (p. 172)
<https://books.google.com/books?id=kCTMFpEcIOwC&lpg=PR9&ots=zwHyTPTvlB&lr&pg=PA188#v=onepage&q&f=false>

: Historically, BSD-derived systems provide the `gethostname` function to return
: only the name of the host.
: ...
: If the host is connected to a TCP/IP network, the host name is normally the
: fully qualified domain name of the host.

Linux in particular seems to have interpreted `gethostname` as
returning a DNS label (maximum of 63 characters) rather than a full
DNS name (up to 213 characters).
Thus it's impossible to represent certain DNS names as the system hostname.

[Maxim Dounin]

Hello!

Post by David Coles
Hi Maxim. Thanks for your comments.

Post by Maxim Dounin
No, thank you. This change makes nginx startup unconditionally
dependant on the DNS availability. This is not something we are
going to accept.

That's reasonable. I based this behaviour off `hostname -f` which also
uses `gethostbyname` for resolving the FQDN.
Note, on most systems this doesn't trigger an actual DNS lookup
provided that hosts canonical name (usually the FQDN) is configured in
`/etc/hosts` (See paragraph 5 of THE FQDN
<https://manpages.debian.org/testing/hostname/hostname.1.en.html>).

The "no DNS lookup on most systems" is not something we can rely
on.

Post by David Coles
On some systems `gethostent` can be used to explicitly walk
`/etc/hosts`, but behaviour is so inconsistent I would avoid it.

Trying to use /etc/hosts will lead to results different from
results of normal name resolution, so this doesn't looks like an
option.

Post by David Coles
How would you feel about trying to resolve the FQDN at nginx startup
and, if unavailable falling back to `$hostname`? This matches the
behaviour of other languages (e.g. Python's `socket.getfqdn()`
<https://docs.python.org/3/library/socket.html#socket.getfqdn>).

Even trying to resolve the FQDN will imply DNS dependency. If DNS
is not availble it will wait for a timeout to expire before
proceeding any further.

Post by David Coles
I could also make this a runtime evaluation, but it would be nice to
only evaluate once as it would often be used in hot paths like logging
and tracing.

Using system resolver at runtime is basically not possible,
because there is no non-blocking interface to system resolver, and
using blocking one will will block the whole worker process during
name resolution - which can take significant time. That's
basically why nginx has resolver in it.

The only option I see is doing name resolution only conditionally,
either when requested by a configuration directive, or when the
variable is actually used in the configuration. But I don't think
this worth the effort - configuring appropriate hostname and/or
using the name directly in nginx configuration would be a better
idea.

[...]

Post by David Coles
Linux in particular seems to have interpreted `gethostname` as
returning a DNS label (maximum of 63 characters) rather than a full
DNS name (up to 213 characters).
Thus it's impossible to represent certain DNS names as the system hostname.

There is more than one Linux distribution, and Debian is just one
of them. There are others which use FQDN:

https://serverfault.com/questions/331936/setting-the-hostname-fqdn-or-short-name

Note well that Debian can use FQDN as well, and "hostname -f" is
smart enough to use such name without looking further.

While length restrictions might make it impossible to represent
some names, this is unlikely a practical problem.