*) test: added first mod_proxy tests in test/modules/proxy that

check some variations on forward, reverse and mixed vhosts
     and also using a unix: domain socket backend.



git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/trunk@1896002 13f79535-47bb-0310-9956-ffa450edef68

test/Makefile.in

@@ -23,4 +23,4 @@ clean:
 	rm -rf gen
 
 distclean:
-	rm -f pytest/config.ini
+	rm -f pyhttpd/config.ini

test/modules/proxy/conftest.py

@@ -0,0 +1,51 @@
+import logging
+import os
+import sys
+import pytest
+
+from .env import ProxyTestEnv
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
+
+
+def pytest_report_header(config, startdir):
+    env = ProxyTestEnv()
+    return "mod_proxy: [apache: {aversion}({prefix})]".format(
+        prefix=env.prefix,
+        aversion=env.get_httpd_version(),
+    )
+
+
+@pytest.fixture(scope="package")
+def env(pytestconfig) -> ProxyTestEnv:
+    level = logging.INFO
+    console = logging.StreamHandler()
+    console.setLevel(level)
+    console.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
+    logging.getLogger('').addHandler(console)
+    logging.getLogger('').setLevel(level=level)
+    env = ProxyTestEnv(pytestconfig=pytestconfig)
+    env.setup_httpd()
+    env.apache_access_log_clear()
+    env.httpd_error_log.clear_log()
+    return env
+
+
+@pytest.fixture(autouse=True, scope="package")
+def _session_scope(env):
+    # we'd like to check the httpd error logs after the test suite has
+    # run to catch anything unusual. For this, we setup the ignore list
+    # of errors and warnings that we do expect.
+    env.httpd_error_log.set_ignored_lognos([
+        'AH01144',  # No protocol handler was valid for the URL
+    ])
+
+    env.httpd_error_log.add_ignored_patterns([
+        #re.compile(r'.*urn:ietf:params:acme:error:.*'),
+    ])
+    yield
+    assert env.apache_stop() == 0
+    errors, warnings = env.httpd_error_log.get_missed()
+    assert (len(errors), len(warnings)) == (0, 0),\
+            f"apache logged {len(errors)} errors and {len(warnings)} warnings: \n"\
+            "{0}\n{1}\n".format("\n".join(errors), "\n".join(warnings))

test/modules/proxy/env.py

@@ -0,0 +1,51 @@
+import inspect
+import logging
+import os
+import re
+import subprocess
+from typing import Dict, Any
+
+from pyhttpd.certs import CertificateSpec
+from pyhttpd.conf import HttpdConf
+from pyhttpd.env import HttpdTestEnv, HttpdTestSetup
+
+log = logging.getLogger(__name__)
+
+
+class ProxyTestSetup(HttpdTestSetup):
+
+    def __init__(self, env: 'HttpdTestEnv'):
+        super().__init__(env=env)
+        self.add_source_dir(os.path.dirname(inspect.getfile(ProxyTestSetup)))
+        self.add_modules(["proxy", "proxy_http"])
+
+
+class ProxyTestEnv(HttpdTestEnv):
+
+    def __init__(self, pytestconfig=None):
+        super().__init__(pytestconfig=pytestconfig)
+        self.add_httpd_conf([
+                         ])
+        self._d_reverse = f"reverse.{self.http_tld}"
+        self._d_forward = f"forward.{self.http_tld}"
+        self._d_mixed = f"mixed.{self.http_tld}"
+
+        self.add_httpd_log_modules(["proxy", "proxy_http"])
+        self.add_cert_specs([
+            CertificateSpec(domains=[
+                self._d_forward, self._d_reverse, self._d_mixed
+            ]),
+            CertificateSpec(domains=[f"noh2.{self.http_tld}"], key_type='rsa2048'),
+        ])
+
+    @property
+    def d_forward(self):
+        return self._d_forward
+
+    @property
+    def d_reverse(self):
+        return self._d_reverse
+
+    @property
+    def d_mixed(self):
+        return self._d_mixed

test/modules/proxy/test_01_http.py

@@ -0,0 +1,70 @@
+import os
+import pytest
+
+from pyhttpd.conf import HttpdConf
+
+
+class TestProxyHttp:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        # setup 3 vhosts on https: for reverse, forward and mixed proxying
+        # setup 3 vhosts on http: with different document roots
+        conf = HttpdConf(env)
+        conf.add("ProxyPreserveHost on")
+        conf.start_vhost(domains=[env.d_reverse], port=env.https_port)
+        conf.add([
+            f"ProxyPass / http://127.0.0.1:{env.http_port}/"
+        ])
+        conf.end_vhost()
+        conf.add_vhost(domains=[env.d_reverse], port=env.http_port, doc_root='htdocs/test1')
+
+        conf.start_vhost(domains=[env.d_forward], port=env.https_port)
+        conf.add([
+            "ProxyRequests on"
+        ])
+        conf.end_vhost()
+        conf.add_vhost(domains=[env.d_forward], port=env.http_port, doc_root='htdocs/test2')
+
+        conf.start_vhost(domains=[env.d_mixed], port=env.https_port)
+        conf.add([
+            f"ProxyPass / http://127.0.0.1:{env.http_port}/",
+            "ProxyRequests on"
+        ])
+        conf.end_vhost()
+        conf.add_vhost(domains=[env.d_mixed], port=env.http_port, doc_root='htdocs')
+        conf.install()
+        assert env.apache_restart() == 0
+
+    @pytest.mark.parametrize(["via", "seen"], [
+        ["reverse", "test1"],
+        ["mixed", "generic"],
+    ])
+    def test_proxy_01_001(self, env, via, seen):
+        # make requests to a reverse proxy https: vhost to the http: vhost
+        # check that we see the document we expect there (host matching worked)
+        r = env.curl_get(f"https://{via}.{env.http_tld}:{env.https_port}/alive.json", 5)
+        assert r.response["status"] == 200
+        assert r.json['host'] == seen
+
+    @pytest.mark.parametrize(["via", "seen"], [
+        ["forward", "test2"],
+        ["mixed", "generic"],
+    ])
+    def test_proxy_01_002(self, env, via, seen):
+        # make requests to a forward proxy https: vhost to the http: vhost
+        # check that we see the document we expect there (host matching worked)
+        # we need to explicitly provide a Host: header since mod_proxy cannot
+        # resolve the name via DNS.
+        domain = f"{via}.{env.http_tld}"
+        r = env.curl_get(f"http://127.0.0.1:{env.http_port}/alive.json", 5, options=[
+            '-H', f"Host: {domain}",
+            '--proxy', f"https://{domain}:{env.https_port}/",
+            '--resolve', f"{domain}:{env.https_port}:127.0.0.1",
+            '--proxy-cacert', f"{env.get_ca_pem_file(domain)}",
+
+        ])
+        assert r.exit_code == 0, f"{r.stdout}{r.stderr}"
+        assert r.response["status"] == 200
+        assert r.json['host'] == seen
+

test/modules/proxy/test_02_unix.py

@@ -0,0 +1,185 @@
+import os
+import re
+import socket
+from threading import Thread
+
+import pytest
+
+from pyhttpd.conf import HttpdConf
+from pyhttpd.result import ExecResult
+
+
+class UDSFaker:
+
+    def __init__(self, path):
+        self._uds_path = path
+        self._done = False
+
+    def start(self):
+        def process(self):
+            self._socket.listen(1)
+            self._process()
+
+        try:
+            os.unlink(self._uds_path)
+        except OSError:
+            if os.path.exists(self._uds_path):
+                raise
+        self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        self._socket.bind(self._uds_path)
+        self._thread = Thread(target=process, args=[self])
+        self._thread.start()
+
+    def stop(self):
+        self._done = True
+        self._socket.close()
+
+    def _process(self):
+        while self._done is False:
+            try:
+                c, client_address = self._socket.accept()
+                try:
+                    data = c.recv(16)
+                    c.sendall("""HTTP/1.1 200 Ok
+Server: UdsFaker
+Content-Type: application/json
+Content-Length: 19
+
+{ "host": "faked" }""".encode())
+                finally:
+                    c.close()
+
+            except ConnectionAbortedError:
+                self._done = True
+
+
+class TestProxyUds:
+
+    @pytest.fixture(autouse=True, scope='class')
+    def _class_scope(self, env):
+        # setup 3 vhosts on https: for reverse, forward and
+        # mixed proxying to a unix: domain socket
+        # We setup a UDSFaker running that returns a fixed response
+        UDS_PATH = f"{env.gen_dir}/proxy_02.sock"
+        TestProxyUds.UDS_PATH = UDS_PATH
+        faker = UDSFaker(path=UDS_PATH)
+        faker.start()
+
+        conf = HttpdConf(env)
+        conf.add("ProxyPreserveHost on")
+        conf.start_vhost(domains=[env.d_reverse], port=env.https_port)
+        conf.add([
+            f"ProxyPass / unix:{UDS_PATH}|http://127.0.0.1:{env.http_port}/"
+        ])
+        conf.end_vhost()
+
+        conf.start_vhost(domains=[env.d_forward], port=env.https_port)
+        conf.add([
+            "ProxyRequests on"
+        ])
+        conf.end_vhost()
+
+        conf.start_vhost(domains=[env.d_mixed], port=env.https_port)
+        conf.add([
+            f"ProxyPass / unix:{UDS_PATH}|http://127.0.0.1:{env.http_port}/",
+            "ProxyRequests on"
+        ])
+        conf.end_vhost()
+        conf.install()
+        assert env.apache_restart() == 0
+        yield
+        faker.stop()
+
+    @pytest.mark.parametrize(["via", "seen"], [
+        ["reverse", "faked"],
+        ["mixed", "faked"],
+    ])
+    def test_proxy_02_001(self, env, via, seen):
+        # make requests to a reverse proxy https: vhost to the http: vhost
+        # check that we see the document we expect there (host matching worked)
+        r = env.curl_get(f"https://{via}.{env.http_tld}:{env.https_port}/alive.json", 5)
+        assert r.response["status"] == 200
+        assert r.json['host'] == seen
+
+    @pytest.mark.parametrize(["via", "seen"], [
+        ["forward", "generic"],
+        ["mixed", "faked"],
+    ])
+    def test_proxy_02_002(self, env, via, seen):
+        # make requests to a forward proxy https: vhost to the http: vhost
+        # check that we see the document we expect there (host matching worked)
+        # we need to explicitly provide a Host: header since mod_proxy cannot
+        # resolve the name via DNS.
+        domain = f"{via}.{env.http_tld}"
+        r = env.curl_get(f"http://127.0.0.1:{env.http_port}/alive.json", 5, options=[
+            '-H', f"Host: {domain}",
+            '--proxy', f"https://{domain}:{env.https_port}/",
+            '--resolve', f"{domain}:{env.https_port}:127.0.0.1",
+            '--proxy-cacert', f"{env.get_ca_pem_file(domain)}",
+
+        ])
+        assert r.exit_code == 0, f"{r.stdout}{r.stderr}"
+        assert r.response["status"] == 200
+        assert r.json['host'] == seen
+
+    @pytest.mark.parametrize(["via", "exp_status"], [
+        ["reverse", 400],
+        ["forward", 500],
+        ["mixed", 500],
+    ])
+    def test_proxy_02_003(self, env, via, exp_status):
+        # make requests to a forward proxy https: vhost and GET
+        # a URL which carries the unix: domain socket.
+        # This needs to fail.
+        domain = f"{via}.{env.http_tld}"
+        r = env.run(args=[
+            'openssl', 's_client', '-connect', f"127.0.0.1:{env.https_port}",
+            '-servername', domain,
+            '-crlf', '-ign_eof',
+            '-CAfile', env.get_ca_pem_file(domain)
+        ], intext=f"""GET unix:{TestProxyUds.UDS_PATH}|http://127.0.0.1:{env.http_port}/alive.json HTTP/1.1
+Host: {domain}
+
+""")
+        assert r.exit_code == 0, f"{r.stdout}{r.stderr}"
+        lines = r.stdout.split('\n')
+        rlines = None
+        for idx, l in enumerate(lines):
+            if l.startswith('HTTP/'):
+                rlines = lines[idx:]
+        assert rlines, f"No response found in: {r.stdout}"
+        r2 = self.parse_response(rlines)
+        assert r2.response
+        assert r2.response['status'] == exp_status
+
+    def parse_response(self, lines) -> ExecResult:
+        exp_body = False
+        exp_stat = True
+        r = ExecResult(args=[], exit_code=0, stdout=b'', stderr=b'')
+        header = {}
+        body = []
+        for line in lines:
+            if exp_stat:
+                m = re.match(r'^(\S+) (\d+) (.*)$', line)
+                assert m, f"first line no HTTP status line: {line}"
+                r.add_response({
+                    "protocol": m.group(1),
+                    "status": int(m.group(2)),
+                    "description": m.group(3),
+                    "body": r.outraw
+                })
+                header = {}
+                exp_stat = False
+                exp_body = False
+            elif re.match(r'^\r?$', line):
+                exp_body = True
+            elif exp_body:
+                body.append(line)
+            else:
+                m = re.match(r'^([^:]+):\s*(.*)$', line)
+                assert m, f"not a header line: {line}"
+                header[m.group(1).lower()] = m.group(2)
+        if r.response:
+            r.response["header"] = header
+            r.response["body"] = body
+        return r

test/pyhttpd/env.py

@@ -570,7 +570,7 @@ class HttpdTestEnv:
         if not isinstance(urls, list):
             urls = [urls]
         u = urlparse(urls[0])
-        assert u.hostname, f"hostname not in url: {urls[0]}"
+        #assert u.hostname, f"hostname not in url: {urls[0]}"
         headerfile = f"{self.gen_dir}/curl.headers.{self._curl_headerfiles_n}"
         self._curl_headerfiles_n += 1
 
@@ -583,12 +583,12 @@ class HttpdTestEnv:
             args.append('--insecure')
         elif options and "--cacert" in options:
             pass
-        else:
+        elif u.hostname:
             ca_pem = self.get_ca_pem_file(u.hostname)
             if ca_pem:
                 args.extend(["--cacert", ca_pem])
 
-        if force_resolve and u.hostname != 'localhost' \
+        if force_resolve and u.hostname and u.hostname != 'localhost' \
                 and u.hostname != self._httpd_addr \
                 and not re.match(r'^(\d+|\[|:).*', u.hostname):
             assert u.port, f"port not in url: {urls[0]}"