Explain email recovery better

Removed misleading renewal messages

.gitignore

@@ -2,9 +2,9 @@
 *.egg-info/
 .eggs/
 build/
-dist/
-/venv/
-/venv3/
+dist*/
+/venv*/
+/kgs/
 /.tox/
 letsencrypt.log
 

.pep8

@@ -0,0 +1,4 @@
+[pep8]
+# E265 block comment should start with '# '
+# E501 line too long (X > 79 characters)
+ignore = E265,E501

.pylintrc

@@ -38,7 +38,7 @@ load-plugins=linter_plugin
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=fixme,locally-disabled,abstract-class-not-used
+disable=fixme,locally-disabled,abstract-class-not-used,bad-continuation,too-few-public-methods,no-self-use,invalid-name
 # abstract-class-not-used cannot be disabled locally (at least in pylint 1.4.1)
 
 
@@ -101,7 +101,7 @@ function-rgx=[a-z_][a-z0-9_]{2,40}$
 function-name-hint=[a-z_][a-z0-9_]{2,40}$
 
 # Regular expression matching correct variable names
-variable-rgx=[a-z_][a-z0-9_]{2,30}$
+variable-rgx=[a-z_][a-z0-9_]{1,30}$
 
 # Naming hint for variable names
 variable-name-hint=[a-z_][a-z0-9_]{2,30}$
@@ -218,7 +218,7 @@ ignore-long-lines=^\s*(# )?<?https?://\S+>?$
 single-line-if-stmt=no
 
 # List of optional constructs for which whitespace checking is disabled
-no-space-check=trailing-comma,dict-separator
+no-space-check=trailing-comma
 
 # Maximum number of lines in a module
 max-module-lines=1250
@@ -228,7 +228,8 @@ max-module-lines=1250
 indent-string='    '
 
 # Number of spaces of indent required inside a hanging or continued line.
-indent-after-paren=4
+# This does something silly/broken...
+#indent-after-paren=4
 
 
 [TYPECHECK]

.travis.yml

@@ -2,11 +2,12 @@ language: python
 
 services:
   - rabbitmq
+  - mysql
 
 # http://docs.travis-ci.com/user/ci-environment/#CI-environment-OS
+# gimme has to be kept in sync with Boulder's Go version setting in .travis.yml
 before_install:
-  - travis_retry sudo ./bootstrap/ubuntu.sh
-  - travis_retry sudo apt-get install --no-install-recommends nginx-light openssl
+  - '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || eval "$(gimme 1.5)"'
 
 # using separate envs with different TOXENVs creates 4x1 Travis build
 # matrix, which allows us to clearly distinguish which component under
@@ -14,16 +15,47 @@ before_install:
 env:
   global:
     - GOPATH=/tmp/go
+    - PATH=$GOPATH/bin:$PATH
   matrix:
-    - TOXENV=py26 BOULDER_INTEGRATION=1
     - TOXENV=py27 BOULDER_INTEGRATION=1
-    - TOXENV=py33
-    - TOXENV=py34
     - TOXENV=lint
     - TOXENV=cover
 
+
+# Only build pushes to the master branch, PRs, and branches beginning with
+# `test-`. This reduces the number of simultaneous Travis runs, which speeds
+# turnaround time on review since there is a cap of 5 simultaneous runs.
+branches:
+  only:
+    - master
+    - /^test-.*$/
+
+sudo: false  # containers
+addons:
+  # make sure simplehttp simple verification works (custom /etc/hosts)
+  hosts:
+    - le.wtf
+  mariadb: "10.0"
+  apt:
+    packages:  # keep in sync with bootstrap/ubuntu.sh and Boulder
+    - lsb-release
+    - python
+    - python-dev
+    - python-virtualenv
+    - gcc
+    - dialog
+    - libaugeas0
+    - libssl-dev
+    - libffi-dev
+    - ca-certificates
+    # For letsencrypt-nginx integration testing
+    - nginx-light
+    - openssl
+    # For Boulder integration testing
+    - rsyslog
+
 install: "travis_retry pip install tox coveralls"
-before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh amqp'
+before_script: '[ "xxx$BOULDER_INTEGRATION" = "xxx" ] || ./tests/boulder-start.sh'
 script: 'travis_retry tox && ([ "xxx$BOULDER_INTEGRATION" = "xxx" ] || (source .tox/$TOXENV/bin/activate && ./tests/boulder-integration.sh))'
 
 after_success: '[ "$TOXENV" == "cover" ] && coveralls'

Dockerfile

@@ -62,5 +62,5 @@ RUN virtualenv --no-site-packages -p python2 /opt/letsencrypt/venv && \
 # bash" and investigate, apply patches, etc.
 
 ENV PATH /opt/letsencrypt/venv/bin:$PATH
-# TODO: is --text really necessary?
-ENTRYPOINT [ "letsencrypt", "--text" ]
+
+ENTRYPOINT [ "letsencrypt" ]

Dockerfile-dev

@@ -0,0 +1,69 @@
+# This Dockerfile builds an image for development.
+FROM ubuntu:trusty
+MAINTAINER Jakub Warmuz <jakub@warmuz.org>
+MAINTAINER William Budington <bill@eff.org>
+MAINTAINER Yan <yan@eff.org>
+
+# Note: this only exposes the port to other docker containers. You
+# still have to bind to 443@host at runtime, as per the ACME spec.
+EXPOSE 443
+
+# TODO: make sure --config-dir and --work-dir cannot be changed
+# through the CLI (letsencrypt-docker wrapper that uses standalone
+# authenticator and text mode only?)
+VOLUME /etc/letsencrypt /var/lib/letsencrypt
+
+WORKDIR /opt/letsencrypt
+
+# no need to mkdir anything:
+# https://docs.docker.com/reference/builder/#copy
+# If <dest> doesn't exist, it is created along with all missing
+# directories in its path.
+
+# TODO: Install non-default Python versions for tox.
+# TODO: Install Apache/Nginx for plugin development.
+COPY bootstrap/ubuntu.sh /opt/letsencrypt/src/
+RUN /opt/letsencrypt/src/ubuntu.sh && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* \
+           /tmp/* \
+           /var/tmp/*
+
+# the above is not likely to change, so by putting it further up the
+# Dockerfile we make sure we cache as much as possible
+
+COPY setup.py README.rst CHANGES.rst MANIFEST.in requirements.txt EULA linter_plugin.py tox.cover.sh tox.ini pep8.travis.sh .pep8 .pylintrc /opt/letsencrypt/src/
+
+# all above files are necessary for setup.py, however, package source
+# code directory has to be copied separately to a subdirectory...
+# https://docs.docker.com/reference/builder/#copy: "If <src> is a
+# directory, the entire contents of the directory are copied,
+# including filesystem metadata. Note: The directory itself is not
+# copied, just its contents." Order again matters, three files are far
+# more likely to be cached than the whole project directory
+
+COPY letsencrypt /opt/letsencrypt/src/letsencrypt/
+COPY acme /opt/letsencrypt/src/acme/
+COPY letsencrypt-apache /opt/letsencrypt/src/letsencrypt-apache/
+COPY letsencrypt-nginx /opt/letsencrypt/src/letsencrypt-nginx/
+COPY letshelp-letsencrypt /opt/letsencrypt/src/letshelp-letsencrypt/
+COPY letsencrypt-compatibility-test /opt/letsencrypt/src/letsencrypt-compatibility-test/
+COPY tests /opt/letsencrypt/src/tests/
+
+RUN virtualenv --no-site-packages -p python2 /opt/letsencrypt/venv && \
+    /opt/letsencrypt/venv/bin/pip install \
+    -r /opt/letsencrypt/src/requirements.txt \
+    -e /opt/letsencrypt/src/acme \
+    -e /opt/letsencrypt/src \
+    -e /opt/letsencrypt/src/letsencrypt-apache \
+    -e /opt/letsencrypt/src/letsencrypt-nginx \
+    -e /opt/letsencrypt/src/letshelp-letsencrypt \
+    -e /opt/letsencrypt/src/letsencrypt-compatibility-test \
+    -e /opt/letsencrypt/src[dev,docs,testing]
+
+# install in editable mode (-e) to save space: it's not possible to
+# "rm -rf /opt/letsencrypt/src" (it's stays in the underlaying image);
+# this might also help in debugging: you can "docker run --entrypoint
+# bash" and investigate, apply patches, etc.
+
+ENV PATH /opt/letsencrypt/venv/bin:$PATH

LICENSE.txt

@@ -1,5 +1,5 @@
-Let's Encrypt:
-Copyright (c) Internet Security Research Group
+Let's Encrypt Python Client
+Copyright (c) Electronic Frontier Foundation and others
 Licensed Apache Version 2.0
 
 Incorporating code from nginxparser

MANIFEST.in

@@ -2,6 +2,8 @@ include requirements.txt
 include README.rst
 include CHANGES.rst
 include CONTRIBUTING.md
+include LICENSE.txt
 include linter_plugin.py
 include letsencrypt/EULA
+recursive-include docs *
 recursive-include letsencrypt/tests/testdata *

README.rst

@@ -1,12 +1,18 @@
 .. notice for github users
 
-Official **documentation**, including `installation instructions`_, is
-available at https://letsencrypt.readthedocs.org.
+Disclaimer
+==========
 
-Generic information about Let's Encrypt project can be found at
-https://letsencrypt.org. Please read `Frequently Asked Questions (FAQ)
-<https://letsencrypt.org/faq/>`_.
+This is a **DEVELOPER PREVIEW** intended for developers and testers only.
 
+**DO NOT RUN THIS CODE ON A PRODUCTION SERVER. IT WILL INSTALL CERTIFICATES
+SIGNED BY A TEST CA, AND WILL CAUSE CERT WARNINGS FOR USERS.**
+
+Browser-trusted certificates will be available in the coming months.
+
+For more information regarding the status of the project, please see
+https://letsencrypt.org. Be sure to checkout the
+`Frequently Asked Questions (FAQ) <https://community.letsencrypt.org/t/frequently-asked-questions-faq/26#topic-title>`_.
 
 About the Let's Encrypt Client
 ==============================
@@ -18,7 +24,7 @@ In short: getting and installing SSL/TLS certificates made easy (`watch demo vid
 The Let's Encrypt Client is a tool to automatically receive and install
 X.509 certificates to enable TLS on servers. The client will
 interoperate with the Let's Encrypt CA which will be issuing browser-trusted
-certificates for free beginning the summer of 2015.
+certificates for free.
 
 It's all automated:
 
@@ -32,7 +38,7 @@ All you need to do to sign a single domain is::
   user@www:~$ sudo letsencrypt -d www.example.org auth
 
 For multiple domains (SAN) use::
-  
+
   user@www:~$ sudo letsencrypt -d www.example.org -d example.org auth
 
 and if you have a compatible web server (Apache or Nginx), Let's Encrypt can
@@ -67,22 +73,13 @@ server automatically!::
 .. _watch demo video: https://www.youtube.com/watch?v=Gas_sSB-5SU
 
 
-Disclaimer
-----------
-
-This is a **DEVELOPER PREVIEW** intended for developers and testers only.
-
-**DO NOT RUN THIS CODE ON A PRODUCTION SERVER. IT WILL INSTALL CERTIFICATES
-SIGNED BY A TEST CA, AND WILL CAUSE CERT WARNINGS FOR USERS.**
-
-
 Current Features
 ----------------
 
 * web servers supported:
 
   - apache/2.x (tested and working on Ubuntu Linux)
-  - nginx/0.8.48+ (tested and mostly working on Ubuntu Linux)
+  - nginx/0.8.48+ (under development)
   - standalone (runs its own webserver to prove you control the domain)
 
 * the private key is generated locally on your system
@@ -99,6 +96,13 @@ Current Features
 * Free and Open Source Software, made with Python.
 
 
+Installation Instructions
+-------------------------
+
+Official **documentation**, including `installation instructions`_, is
+available at https://letsencrypt.readthedocs.org.
+
+
 Links
 -----
 
@@ -112,6 +116,8 @@ Main Website: https://letsencrypt.org/
 
 IRC Channel: #letsencrypt on `Freenode`_
 
+Community: https://community.letsencrypt.org
+
 Mailing list: `client-dev`_ (to subscribe without a Google account, send an
 email to client-dev+subscribe@letsencrypt.org)
 

acme/LICENSE.txt

@@ -0,0 +1,190 @@
+   Copyright 2015 Electronic Frontier Foundation and others
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

acme/MANIFEST.in

@@ -1 +1,3 @@
+include LICENSE.txt
+include README.rst
 recursive-include acme/testdata *

acme/README.rst

@@ -0,0 +1 @@
+ACME protocol implementation for Python

acme/acme/challenges.py

@@ -1,13 +1,9 @@
 """ACME Identifier Validation Challenges."""
-import binascii
 import functools
 import hashlib
 import logging
-import os
 import socket
 
-from cryptography.hazmat.backends import default_backend
-from cryptography import x509
 import OpenSSL
 import requests
 
@@ -29,6 +25,14 @@ class Challenge(jose.TypedJSONObjectWithFields):
     """ACME challenge."""
     TYPES = {}
 
+    @classmethod
+    def from_json(cls, jobj):
+        try:
+            return super(Challenge, cls).from_json(jobj)
+        except jose.UnrecognizedTypeError as error:
+            logger.debug(error)
+            return UnrecognizedChallenge.from_json(jobj)
+
 
 class ContinuityChallenge(Challenge):  # pylint: disable=abstract-method
     """Client validation challenges."""
@@ -46,6 +50,32 @@ class ChallengeResponse(jose.TypedJSONObjectWithFields):
     resource = fields.Resource(resource_type)
 
 
+class UnrecognizedChallenge(Challenge):
+    """Unrecognized challenge.
+
+    ACME specification defines a generic framework for challenges and
+    defines some standard challenges that are implemented in this
+    module. However, other implementations (including peers) might
+    define additional challenge types, which should be ignored if
+    unrecognized.
+
+    :ivar jobj: Original JSON decoded object.
+
+    """
+
+    def __init__(self, jobj):
+        super(UnrecognizedChallenge, self).__init__()
+        object.__setattr__(self, "jobj", jobj)
+
+    def to_partial_json(self):
+        # pylint: disable=no-member
+        return self.jobj
+
+    @classmethod
+    def from_json(cls, jobj):
+        return cls(jobj)
+
+
 @Challenge.register
 class SimpleHTTP(DVChallenge):
     """ACME "simpleHttp" challenge.
@@ -54,43 +84,45 @@ class SimpleHTTP(DVChallenge):
 
     """
     typ = "simpleHttp"
-    token = jose.Field("token")
+
+    TOKEN_SIZE = 128 / 8  # Based on the entropy value from the spec
+    """Minimum size of the :attr:`token` in bytes."""
+
+    # TODO: acme-spec doesn't specify token as base64-encoded value
+    token = jose.Field(
+        "token", encoder=jose.encode_b64jose, decoder=functools.partial(
+            jose.decode_b64jose, size=TOKEN_SIZE, minimum=True))
+
+    @property
+    def good_token(self):  # XXX: @token.decoder
+        """Is `token` good?
+
+        .. todo:: acme-spec wants "It MUST NOT contain any non-ASCII
+           characters", but it should also warrant that it doesn't
+           contain ".." or "/"...
+
+        """
+        # TODO: check that path combined with uri does not go above
+        # URI_ROOT_PATH!
+        return b'..' not in self.token and b'/' not in self.token
 
 
 @ChallengeResponse.register
 class SimpleHTTPResponse(ChallengeResponse):
     """ACME "simpleHttp" challenge response.
 
-    :ivar unicode path:
-    :ivar unicode tls:
+    :ivar bool tls:
 
     """
     typ = "simpleHttp"
-    path = jose.Field("path")
     tls = jose.Field("tls", default=True, omitempty=True)
 
     URI_ROOT_PATH = ".well-known/acme-challenge"
     """URI root path for the server provisioned resource."""
 
-    _URI_TEMPLATE = "{scheme}://{domain}/" + URI_ROOT_PATH + "/{path}"
+    _URI_TEMPLATE = "{scheme}://{domain}/" + URI_ROOT_PATH + "/{token}"
 
-    MAX_PATH_LEN = 25
-    """Maximum allowed `path` length."""
-
-    CONTENT_TYPE = "text/plain"
-
-    @property
-    def good_path(self):
-        """Is `path` good?
-
-        .. todo:: acme-spec: "The value MUST be comprised entirely of
-           characters from the URL-safe alphabet for Base64 encoding
-           [RFC4648]", base64.b64decode ignores those characters
-
-        """
-        # TODO: check that path combined with uri does not go above
-        # URI_ROOT_PATH!
-        return len(self.path) <= 25
+    CONTENT_TYPE = "application/jose+json"
 
     @property
     def scheme(self):
@@ -102,27 +134,91 @@ class SimpleHTTPResponse(ChallengeResponse):
         """Port that the ACME client should be listening for validation."""
         return 443 if self.tls else 80
 
-    def uri(self, domain):
+    def uri(self, domain, chall):
         """Create an URI to the provisioned resource.
 
         Forms an URI to the HTTPS server provisioned resource
         (containing :attr:`~SimpleHTTP.token`).
 
         :param unicode domain: Domain name being verified.
+        :param challenges.SimpleHTTP chall:
 
         """
         return self._URI_TEMPLATE.format(
-            scheme=self.scheme, domain=domain, path=self.path)
+            scheme=self.scheme, domain=domain, token=chall.encode("token"))
 
-    def simple_verify(self, chall, domain, port=None):
+    def gen_resource(self, chall):
+        """Generate provisioned resource.
+
+        :param challenges.SimpleHTTP chall:
+        :rtype: SimpleHTTPProvisionedResource
+
+        """
+        return SimpleHTTPProvisionedResource(token=chall.token, tls=self.tls)
+
+    def gen_validation(self, chall, account_key, alg=jose.RS256, **kwargs):
+        """Generate validation.
+
+        :param challenges.SimpleHTTP chall:
+        :param .JWK account_key: Private account key.
+        :param .JWA alg:
+
+        :returns: `.SimpleHTTPProvisionedResource` signed in `.JWS`
+        :rtype: .JWS
+
+        """
+        return jose.JWS.sign(
+            payload=self.gen_resource(chall).json_dumps(
+                sort_keys=True).encode('utf-8'),
+            key=account_key, alg=alg, **kwargs)
+
+    def check_validation(self, validation, chall, account_public_key):
+        """Check validation.
+
+        :param .JWS validation:
+        :param challenges.SimpleHTTP chall:
+        :type account_public_key:
+            `~cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.dsa.DSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey`
+            wrapped in `.ComparableKey`
+
+        :rtype: bool
+
+        """
+        if not validation.verify(key=account_public_key):
+            return False
+
+        try:
+            resource = SimpleHTTPProvisionedResource.json_loads(
+                validation.payload.decode('utf-8'))
+        except jose.DeserializationError as error:
+            logger.debug(error)
+            return False
+
+        return resource.token == chall.token and resource.tls == self.tls
+
+    def simple_verify(self, chall, domain, account_public_key, port=None):
         """Simple verify.
 
         According to the ACME specification, "the ACME server MUST
         ignore the certificate provided by the HTTPS server", so
         ``requests.get`` is called with ``verify=False``.
 
-        :param .SimpleHTTP chall: Corresponding challenge.
+        :param challenges.SimpleHTTP chall: Corresponding challenge.
         :param unicode domain: Domain name being verified.
+        :param account_public_key: Public key for the key pair
+            being authorized. If ``None`` key verification is not
+            performed!
+        :type account_public_key:
+            `~cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.dsa.DSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey`
+            wrapped in `.ComparableKey`
         :param int port: Port used in the validation.
 
         :returns: ``True`` iff validation is successful, ``False``
@@ -138,76 +234,67 @@ class SimpleHTTPResponse(ChallengeResponse):
                 "Using non-standard port for SimpleHTTP verification: %s", port)
             domain += ":{0}".format(port)
 
-        uri = self.uri(domain)
+        uri = self.uri(domain, chall)
         logger.debug("Verifying %s at %s...", chall.typ, uri)
         try:
             http_response = requests.get(uri, verify=False)
         except requests.exceptions.RequestException as error:
             logger.error("Unable to reach %s: %s", uri, error)
             return False
-        logger.debug(
-            "Received %s. Headers: %s", http_response, http_response.headers)
+        logger.debug("Received %s: %s. Headers: %s", http_response,
+                     http_response.text, http_response.headers)
 
-        good_token = http_response.text == chall.token
-        if not good_token:
-            logger.error(
-                "Unable to verify %s! Expected: %r, returned: %r.",
-                uri, chall.token, http_response.text)
-        # TODO: spec contradicts itself, c.f.
-        # https://github.com/letsencrypt/acme-spec/pull/156/files#r33136438
-        good_ct = self.CONTENT_TYPE == http_response.headers.get(
-            "Content-Type", self.CONTENT_TYPE)
-        return self.good_path and good_ct and good_token
+        if self.CONTENT_TYPE != http_response.headers.get(
+                "Content-Type", self.CONTENT_TYPE):
+            return False
+
+        try:
+            validation = jose.JWS.json_loads(http_response.text)
+        except jose.DeserializationError as error:
+            logger.debug(error)
+            return False
+
+        return self.check_validation(validation, chall, account_public_key)
+
+
+class SimpleHTTPProvisionedResource(jose.JSONObjectWithFields):
+    """SimpleHTTP provisioned resource."""
+    typ = fields.Fixed("type", SimpleHTTP.typ)
+    token = SimpleHTTP._fields["token"]
+    # If the "tls" field is not included in the response, then
+    # validation object MUST have its "tls" field set to "true".
+    tls = jose.Field("tls", omitempty=False)
 
 
 @Challenge.register
 class DVSNI(DVChallenge):
     """ACME "dvsni" challenge.
 
-    :ivar bytes r: Random data, **not** base64-encoded.
-    :ivar bytes nonce: Random data, **not** hex-encoded.
+    :ivar bytes token: Random data, **not** base64-encoded.
 
     """
     typ = "dvsni"
 
-    DOMAIN_SUFFIX = b".acme.invalid"
-    """Domain name suffix."""
-
-    R_SIZE = 32
-    """Required size of the :attr:`r` in bytes."""
-
-    NONCE_SIZE = 16
-    """Required size of the :attr:`nonce` in bytes."""
-
     PORT = 443
     """Port to perform DVSNI challenge."""
 
-    r = jose.Field("r", encoder=jose.encode_b64jose,  # pylint: disable=invalid-name
-                   decoder=functools.partial(jose.decode_b64jose, size=R_SIZE))
-    nonce = jose.Field("nonce", encoder=jose.encode_hex16,
-                       decoder=functools.partial(functools.partial(
-                           jose.decode_hex16, size=NONCE_SIZE)))
+    TOKEN_SIZE = 128 / 8  # Based on the entropy value from the spec
+    """Minimum size of the :attr:`token` in bytes."""
 
-    @property
-    def nonce_domain(self):
-        """Domain name used in SNI.
+    token = jose.Field(
+        "token", encoder=jose.encode_b64jose, decoder=functools.partial(
+            jose.decode_b64jose, size=TOKEN_SIZE, minimum=True))
 
-        :rtype: bytes
+    def gen_response(self, account_key, alg=jose.RS256, **kwargs):
+        """Generate response.
+
+        :param .JWK account_key: Private account key.
+        :rtype: .DVSNIResponse
 
         """
-        return binascii.hexlify(self.nonce) + self.DOMAIN_SUFFIX
-
-    def probe_cert(self, domain, **kwargs):
-        """Probe DVSNI challenge certificate."""
-        host = socket.gethostbyname(domain)
-        logging.debug('%s resolved to %s', domain, host)
-
-        kwargs.setdefault("host", host)
-        kwargs.setdefault("port", self.PORT)
-        kwargs["name"] = self.nonce_domain
-        # TODO: try different methods?
-        # pylint: disable=protected-access
-        return crypto_util._probe_sni(**kwargs)
+        return DVSNIResponse(validation=jose.JWS.sign(
+            payload=self.json_dumps(sort_keys=True).encode('utf-8'),
+            key=account_key, alg=alg, **kwargs))
 
 
 @ChallengeResponse.register
@@ -219,105 +306,138 @@ class DVSNIResponse(ChallengeResponse):
     """
     typ = "dvsni"
 
-    DOMAIN_SUFFIX = DVSNI.DOMAIN_SUFFIX
+    DOMAIN_SUFFIX = b".acme.invalid"
     """Domain name suffix."""
 
-    S_SIZE = 32
-    """Required size of the :attr:`s` in bytes."""
+    PORT = DVSNI.PORT
+    """Port to perform DVSNI challenge."""
 
-    s = jose.Field("s", encoder=jose.encode_b64jose,  # pylint: disable=invalid-name
-                   decoder=functools.partial(jose.decode_b64jose, size=S_SIZE))
+    validation = jose.Field("validation", decoder=jose.JWS.from_json)
 
-    def __init__(self, s=None, *args, **kwargs):
-        s = os.urandom(self.S_SIZE) if s is None else s
-        super(DVSNIResponse, self).__init__(s=s, *args, **kwargs)
-
-    def z(self, chall):  # pylint: disable=invalid-name
-        """Compute the parameter ``z``.
-
-        :param challenge: Corresponding challenge.
-        :type challenge: :class:`DVSNI`
+    @property
+    def z(self):  # pylint: disable=invalid-name
+        """The ``z``  parameter.
 
         :rtype: bytes
 
         """
-        z = hashlib.new("sha256")  # pylint: disable=invalid-name
-        z.update(chall.r)
-        z.update(self.s)
-        return z.hexdigest().encode()
+        # Instance of 'Field' has no 'signature' member
+        # pylint: disable=no-member
+        return hashlib.sha256(self.validation.signature.encode(
+            "signature").encode("utf-8")).hexdigest().encode()
 
-    def z_domain(self, chall):
+    @property
+    def z_domain(self):
         """Domain name for certificate subjectAltName.
 
-        :rtype bytes:
+        :rtype: bytes
 
         """
-        return self.z(chall) + self.DOMAIN_SUFFIX
+        z = self.z  # pylint: disable=invalid-name
+        return z[:32] + b'.' + z[32:] + self.DOMAIN_SUFFIX
 
-    def gen_cert(self, chall, domain, key):
+    @property
+    def chall(self):
+        """Get challenge encoded in the `validation` payload.
+
+        :rtype: challenges.DVSNI
+
+        """
+        # pylint: disable=no-member
+        return DVSNI.json_loads(self.validation.payload.decode('utf-8'))
+
+    def gen_cert(self, key=None, bits=2048):
         """Generate DVSNI certificate.
 
-        :param .DVSNI chall: Corresponding challenge.
-        :param unicode domain:
-        :param OpenSSL.crypto.PKey
+        :param OpenSSL.crypto.PKey key: Optional private key used in
+            certificate generation. If not provided (``None``), then
+            fresh key will be generated.
+        :param int bits: Number of bits for newly generated key.
+
+        :rtype: `tuple` of `OpenSSL.crypto.X509` and
+            `OpenSSL.crypto.PKey`
 
         """
+        if key is None:
+            key = OpenSSL.crypto.PKey()
+            key.generate_key(OpenSSL.crypto.TYPE_RSA, bits)
         return crypto_util.gen_ss_cert(key, [
-            domain, chall.nonce_domain.decode(), self.z_domain(chall).decode()])
+            # z_domain is too big to fit into CN, hence first dummy domain
+            'dummy', self.z_domain.decode()], force_san=True), key
 
-    def simple_verify(self, chall, domain, public_key, **kwargs):
+    def probe_cert(self, domain, **kwargs):
+        """Probe DVSNI challenge certificate.
+
+        :param unicode domain:
+
+        """
+        if "host" not in kwargs:
+            host = socket.gethostbyname(domain)
+            logging.debug('%s resolved to %s', domain, host)
+            kwargs["host"] = host
+
+        kwargs.setdefault("port", self.PORT)
+        kwargs["name"] = self.z_domain
+        # TODO: try different methods?
+        # pylint: disable=protected-access
+        return crypto_util.probe_sni(**kwargs)
+
+    def verify_cert(self, cert):
+        """Verify DVSNI challenge certificate."""
+        # pylint: disable=protected-access
+        sans = crypto_util._pyopenssl_cert_or_req_san(cert)
+        logging.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans)
+        return self.z_domain.decode() in sans
+
+    def simple_verify(self, chall, domain, account_public_key,
+                      cert=None, **kwargs):
         """Simple verify.
 
-        Probes DVSNI certificate and checks it using `verify_cert`;
-        hence all arguments documented in `verify_cert`.
-
-        """
-        try:
-            cert = chall.probe_cert(domain=domain, **kwargs)
-        except errors.Error as error:
-            logger.debug(error, exc_info=True)
-            return False
-        return self.verify_cert(chall, domain, public_key, cert)
-
-    def verify_cert(self, chall, domain, public_key, cert):
-        """Verify DVSNI certificate.
+        Verify ``validation`` using ``account_public_key``, optionally
+        probe DVSNI certificate and check using `verify_cert`.
 
         :param .challenges.DVSNI chall: Corresponding challenge.
         :param str domain: Domain name being validated.
-        :param public_key: Public key for the key pair
-            being authorized. If ``None`` key verification is not
-            performed!
-        :type public_key:
+        :type account_public_key:
             `~cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey`
             or
             `~cryptography.hazmat.primitives.asymmetric.dsa.DSAPublicKey`
             or
             `~cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey`
-            wrapped in `.ComparableKey
-        :param OpenSSL.crypto.X509 cert:
+            wrapped in `.ComparableKey`
+        :param OpenSSL.crypto.X509 cert: Optional certificate. If not
+            provided (``None``) certificate will be retrieved using
+            `probe_cert`.
 
         :returns: ``True`` iff client's control of the domain has been
             verified, ``False`` otherwise.
         :rtype: bool
 
         """
-        # TODO: check "It is a valid self-signed certificate" and
-        # return False if not
-
-        # pylint: disable=protected-access
-        sans = crypto_util._pyopenssl_cert_or_req_san(cert)
-        logging.debug('Certificate %s. SANs: %s', cert.digest('sha1'), sans)
-
-        cert = x509.load_der_x509_certificate(
-            OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert),
-            default_backend())
-
-        if public_key is None:
-            logging.warn('No key verification is performed')
-        elif public_key != jose.ComparableKey(cert.public_key()):
+        # pylint: disable=no-member
+        if not self.validation.verify(key=account_public_key):
             return False
 
-        return domain in sans and self.z_domain(chall).decode() in sans
+        # TODO: it's not checked that payload has exectly 2 fields!
+        try:
+            decoded_chall = self.chall
+        except jose.DeserializationError as error:
+            logger.debug(error, exc_info=True)
+            return False
+
+        if decoded_chall.token != chall.token:
+            logger.debug("Wrong token: expected %r, found %r",
+                         chall.token, decoded_chall.token)
+            return False
+
+        if cert is None:
+            try:
+                cert = self.probe_cert(domain=domain, **kwargs)
+            except errors.Error as error:
+                logger.debug(error, exc_info=True)
+                return False
+
+        return self.verify_cert(cert)
 
 
 @Challenge.register
@@ -347,23 +467,6 @@ class RecoveryContactResponse(ChallengeResponse):
     token = jose.Field("token", omitempty=True)
 
 
-@Challenge.register
-class RecoveryToken(ContinuityChallenge):
-    """ACME "recoveryToken" challenge."""
-    typ = "recoveryToken"
-
-
-@ChallengeResponse.register
-class RecoveryTokenResponse(ChallengeResponse):
-    """ACME "recoveryToken" challenge response.
-
-    :ivar unicode token:
-
-    """
-    typ = "recoveryToken"
-    token = jose.Field("token", omitempty=True)
-
-
 @Challenge.register
 class ProofOfPossession(ContinuityChallenge):
     """ACME "proofOfPossession" challenge.
@@ -445,10 +548,100 @@ class DNS(DVChallenge):
 
     """
     typ = "dns"
-    token = jose.Field("token")
+
+    LABEL = "_acme-challenge"
+    """Label clients prepend to the domain name being validated."""
+
+    TOKEN_SIZE = 128 / 8  # Based on the entropy value from the spec
+    """Minimum size of the :attr:`token` in bytes."""
+
+    token = jose.Field(
+        "token", encoder=jose.encode_b64jose, decoder=functools.partial(
+            jose.decode_b64jose, size=TOKEN_SIZE, minimum=True))
+
+    def gen_validation(self, account_key, alg=jose.RS256, **kwargs):
+        """Generate validation.
+
+        :param .JWK account_key: Private account key.
+        :param .JWA alg:
+
+        :returns: This challenge wrapped in `.JWS`
+        :rtype: .JWS
+
+        """
+        return jose.JWS.sign(
+            payload=self.json_dumps(sort_keys=True).encode('utf-8'),
+            key=account_key, alg=alg, **kwargs)
+
+    def check_validation(self, validation, account_public_key):
+        """Check validation.
+
+        :param JWS validation:
+        :type account_public_key:
+            `~cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.dsa.DSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey`
+            wrapped in `.ComparableKey`
+
+        :rtype: bool
+
+        """
+        if not validation.verify(key=account_public_key):
+            return False
+        try:
+            return self == self.json_loads(
+                validation.payload.decode('utf-8'))
+        except jose.DeserializationError as error:
+            logger.debug("Checking validation for DNS failed: %s", error)
+            return False
+
+    def gen_response(self, account_key, **kwargs):
+        """Generate response.
+
+        :param .JWK account_key: Private account key.
+        :param .JWA alg:
+
+        :rtype: DNSResponse
+
+        """
+        return DNSResponse(validation=self.gen_validation(
+            self, account_key, **kwargs))
+
+    def validation_domain_name(self, name):
+        """Domain name for TXT validation record.
+
+        :param unicode name: Domain name being validated.
+
+        """
+        return "{0}.{1}".format(self.LABEL, name)
 
 
 @ChallengeResponse.register
 class DNSResponse(ChallengeResponse):
-    """ACME "dns" challenge response."""
+    """ACME "dns" challenge response.
+
+    :param JWS validation:
+
+    """
     typ = "dns"
+
+    validation = jose.Field("validation", decoder=jose.JWS.from_json)
+
+    def check_validation(self, chall, account_public_key):
+        """Check validation.
+
+        :param challenges.DNS chall:
+        :type account_public_key:
+            `~cryptography.hazmat.primitives.asymmetric.rsa.RSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.dsa.DSAPublicKey`
+            or
+            `~cryptography.hazmat.primitives.asymmetric.ec.EllipticCurvePublicKey`
+            wrapped in `.ComparableKey`
+
+        :rtype: bool
+
+        """
+        return chall.check_validation(self.validation, account_public_key)

acme/acme/challenges_test.py

@@ -17,15 +17,42 @@ CERT = test_util.load_cert('cert.pem')
 KEY = test_util.load_rsa_private_key('rsa512_key.pem')
 
 
+class ChallengeTest(unittest.TestCase):
+
+    def test_from_json_unrecognized(self):
+        from acme.challenges import Challenge
+        from acme.challenges import UnrecognizedChallenge
+        chall = UnrecognizedChallenge({"type": "foo"})
+        # pylint: disable=no-member
+        self.assertEqual(chall, Challenge.from_json(chall.jobj))
+
+
+class UnrecognizedChallengeTest(unittest.TestCase):
+
+    def setUp(self):
+        from acme.challenges import UnrecognizedChallenge
+        self.jobj = {"type": "foo"}
+        self.chall = UnrecognizedChallenge(self.jobj)
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.jobj, self.chall.to_partial_json())
+
+    def test_from_json(self):
+        from acme.challenges import UnrecognizedChallenge
+        self.assertEqual(
+            self.chall, UnrecognizedChallenge.from_json(self.jobj))
+
+
 class SimpleHTTPTest(unittest.TestCase):
 
     def setUp(self):
         from acme.challenges import SimpleHTTP
         self.msg = SimpleHTTP(
-            token='evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ+PCt92wr+oA')
+            token=jose.decode_b64jose(
+                'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ+PCt92wr+oA'))
         self.jmsg = {
             'type': 'simpleHttp',
-            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ+PCt92wr+oA',
+            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
         }
 
     def test_to_partial_json(self):
@@ -39,56 +66,36 @@ class SimpleHTTPTest(unittest.TestCase):
         from acme.challenges import SimpleHTTP
         hash(SimpleHTTP.from_json(self.jmsg))
 
+    def test_good_token(self):
+        self.assertTrue(self.msg.good_token)
+        self.assertFalse(
+            self.msg.update(token=b'..').good_token)
+
 
 class SimpleHTTPResponseTest(unittest.TestCase):
     # pylint: disable=too-many-instance-attributes
 
     def setUp(self):
         from acme.challenges import SimpleHTTPResponse
-        self.msg_http = SimpleHTTPResponse(
-            path='6tbIMBC5Anhl5bOlWT5ZFA', tls=False)
-        self.msg_https = SimpleHTTPResponse(path='6tbIMBC5Anhl5bOlWT5ZFA')
+        self.msg_http = SimpleHTTPResponse(tls=False)
+        self.msg_https = SimpleHTTPResponse(tls=True)
         self.jmsg_http = {
             'resource': 'challenge',
             'type': 'simpleHttp',
-            'path': '6tbIMBC5Anhl5bOlWT5ZFA',
             'tls': False,
         }
         self.jmsg_https = {
             'resource': 'challenge',
             'type': 'simpleHttp',
-            'path': '6tbIMBC5Anhl5bOlWT5ZFA',
             'tls': True,
         }
 
         from acme.challenges import SimpleHTTP
-        self.chall = SimpleHTTP(token="foo")
-        self.resp_http = SimpleHTTPResponse(path="bar", tls=False)
-        self.resp_https = SimpleHTTPResponse(path="bar", tls=True)
+        self.chall = SimpleHTTP(token=(b"x" * 16))
+        self.resp_http = SimpleHTTPResponse(tls=False)
+        self.resp_https = SimpleHTTPResponse(tls=True)
         self.good_headers = {'Content-Type': SimpleHTTPResponse.CONTENT_TYPE}
 
-    def test_good_path(self):
-        self.assertTrue(self.msg_http.good_path)
-        self.assertTrue(self.msg_https.good_path)
-        self.assertFalse(
-            self.msg_http.update(path=(self.msg_http.path * 10)).good_path)
-
-    def test_scheme(self):
-        self.assertEqual('http', self.msg_http.scheme)
-        self.assertEqual('https', self.msg_https.scheme)
-
-    def test_port(self):
-        self.assertEqual(80, self.msg_http.port)
-        self.assertEqual(443, self.msg_https.port)
-
-    def test_uri(self):
-        self.assertEqual(
-            'http://example.com/.well-known/acme-challenge/'
-            '6tbIMBC5Anhl5bOlWT5ZFA', self.msg_http.uri('example.com'))
-        self.assertEqual(
-            'https://example.com/.well-known/acme-challenge/'
-            '6tbIMBC5Anhl5bOlWT5ZFA', self.msg_https.uri('example.com'))
-
     def test_to_partial_json(self):
         self.assertEqual(self.jmsg_http, self.msg_http.to_partial_json())
         self.assertEqual(self.jmsg_https, self.msg_https.to_partial_json())
@@ -105,34 +112,98 @@ class SimpleHTTPResponseTest(unittest.TestCase):
         hash(SimpleHTTPResponse.from_json(self.jmsg_http))
         hash(SimpleHTTPResponse.from_json(self.jmsg_https))
 
-    @mock.patch("acme.challenges.requests.get")
-    def test_simple_verify_good_token(self, mock_get):
-        for resp in self.resp_http, self.resp_https:
-            mock_get.reset_mock()
-            mock_get.return_value = mock.MagicMock(
-                text=self.chall.token, headers=self.good_headers)
-            self.assertTrue(resp.simple_verify(self.chall, "local"))
-            mock_get.assert_called_once_with(resp.uri("local"), verify=False)
+    def test_scheme(self):
+        self.assertEqual('http', self.msg_http.scheme)
+        self.assertEqual('https', self.msg_https.scheme)
+
+    def test_port(self):
+        self.assertEqual(80, self.msg_http.port)
+        self.assertEqual(443, self.msg_https.port)
+
+    def test_uri(self):
+        self.assertEqual(
+            'http://example.com/.well-known/acme-challenge/'
+            'eHh4eHh4eHh4eHh4eHh4eA', self.msg_http.uri(
+                'example.com', self.chall))
+        self.assertEqual(
+            'https://example.com/.well-known/acme-challenge/'
+            'eHh4eHh4eHh4eHh4eHh4eA', self.msg_https.uri(
+                'example.com', self.chall))
+
+    def test_gen_check_validation(self):
+        account_key = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))
+        self.assertTrue(self.resp_http.check_validation(
+            validation=self.resp_http.gen_validation(self.chall, account_key),
+            chall=self.chall, account_public_key=account_key.public_key()))
+
+    def test_gen_check_validation_wrong_key(self):
+        key1 = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))
+        key2 = jose.JWKRSA.load(test_util.load_vector('rsa1024_key.pem'))
+        self.assertFalse(self.resp_http.check_validation(
+            validation=self.resp_http.gen_validation(self.chall, key1),
+            chall=self.chall, account_public_key=key2.public_key()))
+
+    def test_check_validation_wrong_payload(self):
+        account_key = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))
+        validations = tuple(
+            jose.JWS.sign(payload=payload, alg=jose.RS256, key=account_key)
+            for payload in (b'', b'{}', self.chall.json_dumps().encode('utf-8'),
+                            self.resp_http.json_dumps().encode('utf-8'))
+        )
+        for validation in validations:
+            self.assertFalse(self.resp_http.check_validation(
+                validation=validation, chall=self.chall,
+                account_public_key=account_key.public_key()))
+
+    def test_check_validation_wrong_fields(self):
+        resource = self.resp_http.gen_resource(self.chall)
+        account_key = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))
+        validations = tuple(
+            jose.JWS.sign(payload=bad_resource.json_dumps().encode('utf-8'),
+                          alg=jose.RS256, key=account_key)
+            for bad_resource in (resource.update(tls=True),
+                                 resource.update(token=(b'x' * 20)))
+        )
+        for validation in validations:
+            self.assertFalse(self.resp_http.check_validation(
+                validation=validation, chall=self.chall,
+                account_public_key=account_key.public_key()))
 
     @mock.patch("acme.challenges.requests.get")
-    def test_simple_verify_bad_token(self, mock_get):
+    def test_simple_verify_good_validation(self, mock_get):
+        account_key = jose.JWKRSA.load(test_util.load_vector('rsa512_key.pem'))
+        for resp in self.resp_http, self.resp_https:
+            mock_get.reset_mock()
+            validation = resp.gen_validation(self.chall, account_key)
+            mock_get.return_value = mock.MagicMock(
+                text=validation.json_dumps(), headers=self.good_headers)
+            self.assertTrue(resp.simple_verify(self.chall, "local", None))
+            mock_get.assert_called_once_with(resp.uri(
+                "local", self.chall), verify=False)
+
+    @mock.patch("acme.challenges.requests.get")
+    def test_simple_verify_bad_validation(self, mock_get):
         mock_get.return_value = mock.MagicMock(
-            text=self.chall.token + "!", headers=self.good_headers)
-        self.assertFalse(self.resp_http.simple_verify(self.chall, "local"))
+            text="!", headers=self.good_headers)
+        self.assertFalse(self.resp_http.simple_verify(
+            self.chall, "local", None))
 
     @mock.patch("acme.challenges.requests.get")
     def test_simple_verify_bad_content_type(self, mock_get):
         mock_get().text = self.chall.token
-        self.assertFalse(self.resp_http.simple_verify(self.chall, "local"))
+        self.assertFalse(self.resp_http.simple_verify(
+            self.chall, "local", None))
 
     @mock.patch("acme.challenges.requests.get")
     def test_simple_verify_connection_error(self, mock_get):
         mock_get.side_effect = requests.exceptions.RequestException
-        self.assertFalse(self.resp_http.simple_verify(self.chall, "local"))
+        self.assertFalse(self.resp_http.simple_verify(
+            self.chall, "local", None))
 
     @mock.patch("acme.challenges.requests.get")
     def test_simple_verify_port(self, mock_get):
-        self.resp_http.simple_verify(self.chall, "local", 4430)
+        self.resp_http.simple_verify(
+            self.chall, domain="local", account_public_key=None, port=4430)
         self.assertEqual("local:4430", urllib_parse.urlparse(
             mock_get.mock_calls[0][1][0]).netloc)
 
@@ -142,19 +213,12 @@ class DVSNITest(unittest.TestCase):
     def setUp(self):
         from acme.challenges import DVSNI
         self.msg = DVSNI(
-            r=b"O*\xb4-\xad\xec\x95>\xed\xa9\r0\x94\xe8\x97\x9c&6"
-              b"\xbf'\xb3\xed\x9a9nX\x0f'\\m\xe7\x12",
-            nonce=b'\xa8-_\xf8\xeft\r\x12\x88\x1fm<"w\xab.')
+            token=jose.b64decode('a82d5ff8ef740d12881f6d3c2277ab2e'))
         self.jmsg = {
             'type': 'dvsni',
-            'r': 'Tyq0La3slT7tqQ0wlOiXnCY2vyez7Zo5blgPJ1xt5xI',
-            'nonce': 'a82d5ff8ef740d12881f6d3c2277ab2e',
+            'token': 'a82d5ff8ef740d12881f6d3c2277ab2e',
         }
 
-    def test_nonce_domain(self):
-        self.assertEqual(b'a82d5ff8ef740d12881f6d3c2277ab2e.acme.invalid',
-                         self.msg.nonce_domain)
-
     def test_to_partial_json(self):
         self.assertEqual(self.jmsg, self.msg.to_partial_json())
 
@@ -166,27 +230,76 @@ class DVSNITest(unittest.TestCase):
         from acme.challenges import DVSNI
         hash(DVSNI.from_json(self.jmsg))
 
-    def test_from_json_invalid_r_length(self):
+    def test_from_json_invalid_token_length(self):
         from acme.challenges import DVSNI
-        self.jmsg['r'] = 'abcd'
+        self.jmsg['token'] = jose.encode_b64jose(b'abcd')
         self.assertRaises(
             jose.DeserializationError, DVSNI.from_json, self.jmsg)
 
-    def test_from_json_invalid_nonce_length(self):
+    def test_gen_response(self):
+        key = jose.JWKRSA(key=KEY)
         from acme.challenges import DVSNI
-        self.jmsg['nonce'] = 'abcd'
-        self.assertRaises(
-            jose.DeserializationError, DVSNI.from_json, self.jmsg)
+        self.assertEqual(self.msg, DVSNI.json_loads(
+            self.msg.gen_response(key).validation.payload.decode()))
+
+
+class DVSNIResponseTest(unittest.TestCase):
+    # pylint: disable=too-many-instance-attributes
+
+    def setUp(self):
+        self.key = jose.JWKRSA(key=KEY)
+
+        from acme.challenges import DVSNI
+        self.chall = DVSNI(
+            token=jose.b64decode(b'a82d5ff8ef740d12881f6d3c2277ab2e'))
+
+        from acme.challenges import DVSNIResponse
+        self.validation = jose.JWS.sign(
+            payload=self.chall.json_dumps(sort_keys=True).encode(),
+            key=self.key, alg=jose.RS256)
+        self.msg = DVSNIResponse(validation=self.validation)
+        self.jmsg_to = {
+            'resource': 'challenge',
+            'type': 'dvsni',
+            'validation': self.validation,
+        }
+        self.jmsg_from = {
+            'resource': 'challenge',
+            'type': 'dvsni',
+            'validation': self.validation.to_json(),
+        }
+
+        # pylint: disable=invalid-name
+        label1 = b'e2df3498860637c667fedadc5a8494ec'
+        label2 = b'09dcc75553c9b3bd73662b50e71b1e42'
+        self.z = label1 + label2
+        self.z_domain = label1 + b'.' + label2 + b'.acme.invalid'
+        self.domain = 'foo.com'
+
+    def test_z_and_domain(self):
+        self.assertEqual(self.z, self.msg.z)
+        self.assertEqual(self.z_domain, self.msg.z_domain)
+
+    def test_to_partial_json(self):
+        self.assertEqual(self.jmsg_to, self.msg.to_partial_json())
+
+    def test_from_json(self):
+        from acme.challenges import DVSNIResponse
+        self.assertEqual(self.msg, DVSNIResponse.from_json(self.jmsg_from))
+
+    def test_from_json_hashable(self):
+        from acme.challenges import DVSNIResponse
+        hash(DVSNIResponse.from_json(self.jmsg_from))
 
     @mock.patch('acme.challenges.socket.gethostbyname')
-    @mock.patch('acme.challenges.crypto_util._probe_sni')
+    @mock.patch('acme.challenges.crypto_util.probe_sni')
     def test_probe_cert(self, mock_probe_sni, mock_gethostbyname):
         mock_gethostbyname.return_value = '127.0.0.1'
         self.msg.probe_cert('foo.com')
         mock_gethostbyname.assert_called_once_with('foo.com')
         mock_probe_sni.assert_called_once_with(
             host='127.0.0.1', port=self.msg.PORT,
-            name=b'a82d5ff8ef740d12881f6d3c2277ab2e.acme.invalid')
+            name=self.z_domain)
 
         self.msg.probe_cert('foo.com', host='8.8.8.8')
         mock_probe_sni.assert_called_with(
@@ -203,88 +316,54 @@ class DVSNITest(unittest.TestCase):
         self.msg.probe_cert('foo.com', name=b'xxx')
         mock_probe_sni.assert_called_with(
             host=mock.ANY, port=mock.ANY,
-            name=b'a82d5ff8ef740d12881f6d3c2277ab2e.acme.invalid')
+            name=self.z_domain)
 
+    def test_gen_verify_cert(self):
+        key1 = test_util.load_pyopenssl_private_key('rsa512_key.pem')
+        cert, key2 = self.msg.gen_cert(key1)
+        self.assertEqual(key1, key2)
+        self.assertTrue(self.msg.verify_cert(cert))
 
-class DVSNIResponseTest(unittest.TestCase):
+    def test_gen_verify_cert_gen_key(self):
+        cert, key = self.msg.gen_cert()
+        self.assertTrue(isinstance(key, OpenSSL.crypto.PKey))
+        self.assertTrue(self.msg.verify_cert(cert))
 
-    def setUp(self):
-        from acme.challenges import DVSNIResponse
-        # pylint: disable=invalid-name
-        s = '9dbjsl3gTAtOnEtKFEmhS6Mj-ajNjDcOmRkp3Lfzm3c'
-        self.msg = DVSNIResponse(s=jose.decode_b64jose(s))
-        self.jmsg = {
-            'resource': 'challenge',
-            'type': 'dvsni',
-            's': s,
-        }
+    def test_verify_bad_cert(self):
+        self.assertFalse(self.msg.verify_cert(test_util.load_cert('cert.pem')))
 
-        from acme.challenges import DVSNI
-        self.chall = DVSNI(
-            r=jose.decode_b64jose('Tyq0La3slT7tqQ0wlOiXnCY2vyez7Zo5blgPJ1xt5xI'),
-            nonce=jose.decode_b64jose('a82d5ff8ef740d12881f6d3c2277ab2e'))
-        self.z = (b'38e612b0397cc2624a07d351d7ef50e4'
-                  b'6134c0213d9ed52f7d7c611acaeed41b')
-        self.domain = 'foo.com'
-        self.key = test_util.load_pyopenssl_private_key('rsa512_key.pem')
-        self.public_key = test_util.load_rsa_private_key(
-            'rsa512_key.pem').public_key()
+    def test_simple_verify_wrong_account_key(self):
+        self.assertFalse(self.msg.simple_verify(
+            self.chall, self.domain, jose.JWKRSA.load(
+                test_util.load_vector('rsa256_key.pem')).public_key()))
 
-    def test_z_and_domain(self):
-        # pylint: disable=invalid-name
-        self.assertEqual(self.z, self.msg.z(self.chall))
-        self.assertEqual(
-            self.z + b'.acme.invalid', self.msg.z_domain(self.chall))
+    def test_simple_verify_wrong_payload(self):
+        for payload in b'', b'{}':
+            msg = self.msg.update(validation=jose.JWS.sign(
+                payload=payload, key=self.key, alg=jose.RS256))
+            self.assertFalse(msg.simple_verify(
+                self.chall, self.domain, self.key.public_key()))
 
-    def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+    def test_simple_verify_wrong_token(self):
+        msg = self.msg.update(validation=jose.JWS.sign(
+            payload=self.chall.update(token=(b'b' * 20)).json_dumps().encode(),
+            key=self.key, alg=jose.RS256))
+        self.assertFalse(msg.simple_verify(
+            self.chall, self.domain, self.key.public_key()))
 
-    def test_from_json(self):
-        from acme.challenges import DVSNIResponse
-        self.assertEqual(self.msg, DVSNIResponse.from_json(self.jmsg))
-
-    def test_from_json_hashable(self):
-        from acme.challenges import DVSNIResponse
-        hash(DVSNIResponse.from_json(self.jmsg))
-
-    @mock.patch('acme.challenges.DVSNIResponse.verify_cert')
+    @mock.patch('acme.challenges.DVSNIResponse.verify_cert', autospec=True)
     def test_simple_verify(self, mock_verify_cert):
-        chall = mock.Mock()
-        chall.probe_cert.return_value = mock.sentinel.cert
-        mock_verify_cert.return_value = 'x'
-        self.assertEqual('x', self.msg.simple_verify(
-            chall, mock.sentinel.domain, mock.sentinel.key))
-        chall.probe_cert.assert_called_once_with(domain=mock.sentinel.domain)
-        self.msg.verify_cert.assert_called_once_with(
-            chall, mock.sentinel.domain, mock.sentinel.key,
-            mock.sentinel.cert)
+        mock_verify_cert.return_value = mock.sentinel.verification
+        self.assertEqual(mock.sentinel.verification, self.msg.simple_verify(
+            self.chall, self.domain, self.key.public_key(),
+            cert=mock.sentinel.cert))
+        mock_verify_cert.assert_called_once_with(self.msg, mock.sentinel.cert)
 
     def test_simple_verify_false_on_probe_error(self):
         chall = mock.Mock()
         chall.probe_cert.side_effect = errors.Error
         self.assertFalse(self.msg.simple_verify(
-            chall=chall, domain=None, public_key=None))
-
-    def test_gen_verify_cert_postive_no_key(self):
-        cert = self.msg.gen_cert(self.chall, self.domain, self.key)
-        self.assertTrue(self.msg.verify_cert(
-            self.chall, self.domain, public_key=None, cert=cert))
-
-    def test_gen_verify_cert_postive_with_key(self):
-        cert = self.msg.gen_cert(self.chall, self.domain, self.key)
-        self.assertTrue(self.msg.verify_cert(
-            self.chall, self.domain, public_key=self.public_key, cert=cert))
-
-    def test_gen_verify_cert_negative_with_wrong_key(self):
-        cert = self.msg.gen_cert(self.chall, self.domain, self.key)
-        key = test_util.load_rsa_private_key('rsa256_key.pem').public_key()
-        self.assertFalse(self.msg.verify_cert(
-            self.chall, self.domain, public_key=key, cert=cert))
-
-    def test_gen_verify_cert_negative(self):
-        cert = self.msg.gen_cert(self.chall, self.domain + 'x', self.key)
-        self.assertFalse(self.msg.verify_cert(
-            self.chall, self.domain, public_key=None, cert=cert))
+            self.chall, self.domain, self.key.public_key()))
 
 
 class RecoveryContactTest(unittest.TestCase):
@@ -297,9 +376,9 @@ class RecoveryContactTest(unittest.TestCase):
             contact='c********n@example.com')
         self.jmsg = {
             'type': 'recoveryContact',
-            'activationURL' : 'https://example.ca/sendrecovery/a5bd99383fb0',
-            'successURL' : 'https://example.ca/confirmrecovery/bb1b9928932',
-            'contact' : 'c********n@example.com',
+            'activationURL': 'https://example.ca/sendrecovery/a5bd99383fb0',
+            'successURL': 'https://example.ca/confirmrecovery/bb1b9928932',
+            'contact': 'c********n@example.com',
         }
 
     def test_to_partial_json(self):
@@ -360,58 +439,6 @@ class RecoveryContactResponseTest(unittest.TestCase):
         self.assertEqual(self.jmsg, msg.to_partial_json())
 
 
-class RecoveryTokenTest(unittest.TestCase):
-
-    def setUp(self):
-        from acme.challenges import RecoveryToken
-        self.msg = RecoveryToken()
-        self.jmsg = {'type': 'recoveryToken'}
-
-    def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
-
-    def test_from_json(self):
-        from acme.challenges import RecoveryToken
-        self.assertEqual(self.msg, RecoveryToken.from_json(self.jmsg))
-
-    def test_from_json_hashable(self):
-        from acme.challenges import RecoveryToken
-        hash(RecoveryToken.from_json(self.jmsg))
-
-
-class RecoveryTokenResponseTest(unittest.TestCase):
-
-    def setUp(self):
-        from acme.challenges import RecoveryTokenResponse
-        self.msg = RecoveryTokenResponse(token='23029d88d9e123e')
-        self.jmsg = {
-            'resource': 'challenge',
-            'type': 'recoveryToken',
-            'token': '23029d88d9e123e'
-        }
-
-    def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
-
-    def test_from_json(self):
-        from acme.challenges import RecoveryTokenResponse
-        self.assertEqual(
-            self.msg, RecoveryTokenResponse.from_json(self.jmsg))
-
-    def test_from_json_hashable(self):
-        from acme.challenges import RecoveryTokenResponse
-        hash(RecoveryTokenResponse.from_json(self.jmsg))
-
-    def test_json_without_optionals(self):
-        del self.jmsg['token']
-
-        from acme.challenges import RecoveryTokenResponse
-        msg = RecoveryTokenResponse.from_json(self.jmsg)
-
-        self.assertTrue(msg.token is None)
-        self.assertEqual(self.jmsg, msg.to_partial_json())
-
-
 class ProofOfPossessionHintsTest(unittest.TestCase):
 
     def setUp(self):
@@ -569,9 +596,15 @@ class ProofOfPossessionResponseTest(unittest.TestCase):
 class DNSTest(unittest.TestCase):
 
     def setUp(self):
+        self.account_key = jose.JWKRSA.load(
+            test_util.load_vector('rsa512_key.pem'))
         from acme.challenges import DNS
-        self.msg = DNS(token='17817c66b60ce2e4012dfad92657527a')
-        self.jmsg = {'type': 'dns', 'token': '17817c66b60ce2e4012dfad92657527a'}
+        self.msg = DNS(token=jose.b64decode(
+            b'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'))
+        self.jmsg = {
+            'type': 'dns',
+            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
+        }
 
     def test_to_partial_json(self):
         self.assertEqual(self.jmsg, self.msg.to_partial_json())
@@ -584,27 +617,84 @@ class DNSTest(unittest.TestCase):
         from acme.challenges import DNS
         hash(DNS.from_json(self.jmsg))
 
+    def test_gen_check_validation(self):
+        self.assertTrue(self.msg.check_validation(
+            self.msg.gen_validation(self.account_key),
+            self.account_key.public_key()))
+
+    def test_gen_check_validation_wrong_key(self):
+        key2 = jose.JWKRSA.load(test_util.load_vector('rsa1024_key.pem'))
+        self.assertFalse(self.msg.check_validation(
+            self.msg.gen_validation(self.account_key), key2.public_key()))
+
+    def test_check_validation_wrong_payload(self):
+        validations = tuple(
+            jose.JWS.sign(payload=payload, alg=jose.RS256, key=self.account_key)
+            for payload in (b'', b'{}')
+        )
+        for validation in validations:
+            self.assertFalse(self.msg.check_validation(
+                validation, self.account_key.public_key()))
+
+    def test_check_validation_wrong_fields(self):
+        bad_validation = jose.JWS.sign(
+            payload=self.msg.update(token=b'x' * 20).json_dumps().encode('utf-8'),
+            alg=jose.RS256, key=self.account_key)
+        self.assertFalse(self.msg.check_validation(
+            bad_validation, self.account_key.public_key()))
+
+    def test_gen_response(self):
+        with mock.patch('acme.challenges.DNS.gen_validation') as mock_gen:
+            mock_gen.return_value = mock.sentinel.validation
+            response = self.msg.gen_response(self.account_key)
+        from acme.challenges import DNSResponse
+        self.assertTrue(isinstance(response, DNSResponse))
+        self.assertEqual(response.validation, mock.sentinel.validation)
+
+    def test_validation_domain_name(self):
+        self.assertEqual(
+            '_acme-challenge.le.wtf', self.msg.validation_domain_name('le.wtf'))
+
 
 class DNSResponseTest(unittest.TestCase):
 
     def setUp(self):
+        self.key = jose.JWKRSA(key=KEY)
+
+        from acme.challenges import DNS
+        self.chall = DNS(token=jose.b64decode(
+            b"evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA"))
+        self.validation = jose.JWS.sign(
+            payload=self.chall.json_dumps(sort_keys=True).encode(),
+            key=self.key, alg=jose.RS256)
+
         from acme.challenges import DNSResponse
-        self.msg = DNSResponse()
-        self.jmsg = {
+        self.msg = DNSResponse(validation=self.validation)
+        self.jmsg_to = {
             'resource': 'challenge',
             'type': 'dns',
+            'validation': self.validation,
+        }
+        self.jmsg_from = {
+            'resource': 'challenge',
+            'type': 'dns',
+            'validation': self.validation.to_json(),
         }
 
     def test_to_partial_json(self):
-        self.assertEqual(self.jmsg, self.msg.to_partial_json())
+        self.assertEqual(self.jmsg_to, self.msg.to_partial_json())
 
     def test_from_json(self):
         from acme.challenges import DNSResponse
-        self.assertEqual(self.msg, DNSResponse.from_json(self.jmsg))
+        self.assertEqual(self.msg, DNSResponse.from_json(self.jmsg_from))
 
     def test_from_json_hashable(self):
         from acme.challenges import DNSResponse
-        hash(DNSResponse.from_json(self.jmsg))
+        hash(DNSResponse.from_json(self.jmsg_from))
+
+    def test_check_validation(self):
+        self.assertTrue(
+            self.msg.check_validation(self.chall, self.key.public_key()))
 
 
 if __name__ == '__main__':

acme/acme/client.py

@@ -4,11 +4,12 @@ import heapq
 import logging
 import time
 
+import six
 from six.moves import http_client  # pylint: disable=import-error
 
 import OpenSSL
 import requests
-import six
+import sys
 import werkzeug
 
 from acme import errors
@@ -19,8 +20,9 @@ from acme import messages
 
 logger = logging.getLogger(__name__)
 
+# Python does not validate certificates by default before version 2.7.9
 # https://urllib3.readthedocs.org/en/latest/security.html#insecureplatformwarning
-if six.PY2:
+if sys.version_info < (2, 7, 9):  # pragma: no cover
     requests.packages.urllib3.contrib.pyopenssl.inject_into_urllib3()
 
 
@@ -31,7 +33,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
        Clean up raised error types hierarchy, document, and handle (wrap)
        instances of `.DeserializationError` raised in `from_json()`.
 
-    :ivar str new_reg_uri: Location of new-reg
+    :ivar messages.Directory directory:
     :ivar key: `.JWK` (private)
     :ivar alg: `.JWASignature`
     :ivar bool verify_ssl: Verify SSL certificates?
@@ -42,12 +44,23 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
     """
     DER_CONTENT_TYPE = 'application/pkix-cert'
 
-    def __init__(self, new_reg_uri, key, alg=jose.RS256,
-                 verify_ssl=True, net=None):
-        self.new_reg_uri = new_reg_uri
+    def __init__(self, directory, key, alg=jose.RS256, verify_ssl=True,
+                 net=None):
+        """Initialize.
+
+        :param directory: Directory Resource (`.messages.Directory`) or
+            URI from which the resource will be downloaded.
+
+        """
         self.key = key
         self.net = ClientNetwork(key, alg, verify_ssl) if net is None else net
 
+        if isinstance(directory, six.string_types):
+            self.directory = messages.Directory.from_json(
+                self.net.get(directory).json())
+        else:
+            self.directory = directory
+
     @classmethod
     def _regr_from_response(cls, response, uri=None, new_authzr_uri=None,
                             terms_of_service=None):
@@ -81,7 +94,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         new_reg = messages.NewRegistration() if new_reg is None else new_reg
         assert isinstance(new_reg, messages.NewRegistration)
 
-        response = self.net.post(self.new_reg_uri, new_reg)
+        response = self.net.post(self.directory[new_reg], new_reg)
         # TODO: handle errors
         assert response.status_code == http_client.CREATED
 
@@ -94,18 +107,8 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
 
         return regr
 
-    def update_registration(self, regr):
-        """Update registration.
-
-        :pram regr: Registration Resource.
-        :type regr: `.RegistrationResource`
-
-        :returns: Updated Registration Resource.
-        :rtype: `.RegistrationResource`
-
-        """
-        response = self.net.post(
-            regr.uri, messages.UpdateRegistration(**dict(regr.body)))
+    def _send_recv_regr(self, regr, body):
+        response = self.net.post(regr.uri, body)
 
         # TODO: Boulder returns httplib.ACCEPTED
         #assert response.status_code == httplib.OK
@@ -113,13 +116,37 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         # TODO: Boulder does not set Location or Link on update
         # (c.f. acme-spec #94)
 
-        updated_regr = self._regr_from_response(
+        return self._regr_from_response(
             response, uri=regr.uri, new_authzr_uri=regr.new_authzr_uri,
             terms_of_service=regr.terms_of_service)
+
+    def update_registration(self, regr, update=None):
+        """Update registration.
+
+        :param messages.RegistrationResource regr: Registration Resource.
+        :param messages.Registration update: Updated body of the
+            resource. If not provided, body will be taken from `regr`.
+
+        :returns: Updated Registration Resource.
+        :rtype: `.RegistrationResource`
+
+        """
+        update = regr.body if update is None else update
+        updated_regr = self._send_recv_regr(
+            regr, body=messages.UpdateRegistration(**dict(update)))
         if updated_regr != regr:
             raise errors.UnexpectedUpdate(regr)
         return updated_regr
 
+    def query_registration(self, regr):
+        """Query server about registration.
+
+        :param messages.RegistrationResource: Existing Registration
+            Resource.
+
+        """
+        return self._send_recv_regr(regr, messages.UpdateRegistration())
+
     def agree_to_tos(self, regr):
         """Agree to the terms-of-service.
 
@@ -275,8 +302,7 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         logger.debug("Requesting issuance...")
 
         # TODO: assert len(authzrs) == number of SANs
-        req = messages.CertificateRequest(
-            csr=csr, authorizations=tuple(authzr.uri for authzr in authzrs))
+        req = messages.CertificateRequest(csr=csr)
 
         content_type = self.DER_CONTENT_TYPE  # TODO: add 'cert_type 'argument
         response = self.net.post(
@@ -403,20 +429,34 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         # respond with status code 403 (Forbidden)
         return self.check_cert(certr)
 
-    def fetch_chain(self, certr):
+    def fetch_chain(self, certr, max_length=10):
         """Fetch chain for certificate.
 
-        :param certr: Certificate Resource
-        :type certr: `.CertificateResource`
+        :param .CertificateResource certr: Certificate Resource
+        :param int max_length: Maximum allowed length of the chain.
+            Note that each element in the certificate requires new
+            ``HTTP GET`` request, and the length of the chain is
+            controlled by the ACME CA.
 
-        :returns: Certificate chain, or `None` if no "up" Link was provided.
-        :rtype: `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
+        :raises errors.Error: if recursion exceeds `max_length`
+
+        :returns: Certificate chain for the Certificate Resource. It is
+            a list ordered so that the first element is a signer of the
+            certificate from Certificate Resource. Will be empty if
+            ``cert_chain_uri`` is ``None``.
+        :rtype: `list` of `OpenSSL.crypto.X509` wrapped in `.ComparableX509`
 
         """
-        if certr.cert_chain_uri is not None:
-            return self._get_cert(certr.cert_chain_uri)[1]
-        else:
-            return None
+        chain = []
+        uri = certr.cert_chain_uri
+        while uri is not None and len(chain) < max_length:
+            response, cert = self._get_cert(uri)
+            uri = response.links.get('up', {}).get('url')
+            chain.append(cert)
+        if uri is not None:
+            raise errors.Error(
+                "Recursion limit reached. Didn't get {0}".format(uri))
+        return chain
 
     def revoke(self, cert):
         """Revoke certificate.
@@ -427,8 +467,9 @@ class Client(object):  # pylint: disable=too-many-instance-attributes
         :raises .ClientError: If revocation is unsuccessful.
 
         """
-        response = self.net.post(messages.Revocation.url(self.new_reg_uri),
-                                 messages.Revocation(certificate=cert))
+        response = self.net.post(self.directory[messages.Revocation],
+                                 messages.Revocation(certificate=cert),
+                                 content_type=None)
         if response.status_code != http_client.OK:
             raise errors.ClientError(
                 'Successful revocation must return HTTP OK status')
@@ -534,7 +575,8 @@ class ClientNetwork(object):
 
 
         """
-        logging.debug('Sending %s request to %s', method, url)
+        logging.debug('Sending %s request to %s. args: %r, kwargs: %r',
+                      method, url, args, kwargs)
         kwargs['verify'] = self.verify_ssl
         response = requests.request(method, url, *args, **kwargs)
         logging.debug('Received %s. Headers: %s. Content: %r',
@@ -545,7 +587,7 @@ class ClientNetwork(object):
         """Send HEAD request without checking the response.
 
         Note, that `_check_response` is not called, as it is expected
-        that status code other than successfuly 2xx will be returned, or
+        that status code other than successfully 2xx will be returned, or
         messages2.Error will be raised by the server.
 
         """

acme/acme/client_test.py

@@ -33,10 +33,14 @@ class ClientTest(unittest.TestCase):
         self.net.post.return_value = self.response
         self.net.get.return_value = self.response
 
+        self.directory = messages.Directory({
+            messages.NewRegistration: 'https://www.letsencrypt-demo.org/acme/new-reg',
+            messages.Revocation: 'https://www.letsencrypt-demo.org/acme/revoke-cert',
+        })
+
         from acme.client import Client
         self.client = Client(
-            new_reg_uri='https://www.letsencrypt-demo.org/acme/new-reg',
-            key=KEY, alg=jose.RS256, net=self.net)
+            directory=self.directory, key=KEY, alg=jose.RS256, net=self.net)
 
         self.identifier = messages.Identifier(
             typ=messages.IDENTIFIER_FQDN, value='example.com')
@@ -44,7 +48,7 @@ class ClientTest(unittest.TestCase):
         # Registration
         self.contact = ('mailto:cert-admin@example.com', 'tel:+12025551212')
         reg = messages.Registration(
-            contact=self.contact, key=KEY.public_key(), recovery_token='t')
+            contact=self.contact, key=KEY.public_key())
         self.new_reg = messages.NewRegistration(**dict(reg))
         self.regr = messages.RegistrationResource(
             body=reg, uri='https://www.letsencrypt-demo.org/acme/reg/1',
@@ -55,7 +59,8 @@ class ClientTest(unittest.TestCase):
         authzr_uri = 'https://www.letsencrypt-demo.org/acme/authz/1'
         challb = messages.ChallengeBody(
             uri=(authzr_uri + '/1'), status=messages.STATUS_VALID,
-            chall=challenges.DNS(token='foo'))
+            chall=challenges.DNS(token=jose.b64decode(
+                'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA')))
         self.challr = messages.ChallengeResource(
             body=challb, authzr_uri=authzr_uri)
         self.authz = messages.Authorization(
@@ -72,6 +77,13 @@ class ClientTest(unittest.TestCase):
             uri='https://www.letsencrypt-demo.org/acme/cert/1',
             cert_chain_uri='https://www.letsencrypt-demo.org/ca')
 
+    def test_init_downloads_directory(self):
+        uri = 'http://www.letsencrypt-demo.org/directory'
+        from acme.client import Client
+        self.client = Client(
+            directory=uri, key=KEY, alg=jose.RS256, net=self.net)
+        self.net.get.assert_called_once_with(uri)
+
     def test_register(self):
         # "Instance of 'Field' has no to_json/update member" bug:
         # pylint: disable=no-member
@@ -111,6 +123,10 @@ class ClientTest(unittest.TestCase):
         self.assertRaises(
             errors.UnexpectedUpdate, self.client.update_registration, self.regr)
 
+    def test_query_registration(self):
+        self.response.json.return_value = self.regr.body.to_json()
+        self.assertEqual(self.regr, self.client.query_registration(self.regr))
+
     def test_agree_to_tos(self):
         self.client.update_registration = mock.Mock()
         self.client.agree_to_tos(self.regr)
@@ -151,7 +167,7 @@ class ClientTest(unittest.TestCase):
         self.response.links['up'] = {'url': self.challr.authzr_uri}
         self.response.json.return_value = self.challr.body.to_json()
 
-        chall_response = challenges.DNSResponse()
+        chall_response = challenges.DNSResponse(validation=None)
 
         self.client.answer_challenge(self.challr.body, chall_response)
 
@@ -160,8 +176,9 @@ class ClientTest(unittest.TestCase):
                           self.challr.body.update(uri='foo'), chall_response)
 
     def test_answer_challenge_missing_next(self):
-        self.assertRaises(errors.ClientError, self.client.answer_challenge,
-                          self.challr.body, challenges.DNSResponse())
+        self.assertRaises(
+            errors.ClientError, self.client.answer_challenge,
+            self.challr.body, challenges.DNSResponse(validation=None))
 
     def test_retry_after_date(self):
         self.response.headers['Retry-After'] = 'Fri, 31 Dec 1999 23:59:59 GMT'
@@ -331,21 +348,39 @@ class ClientTest(unittest.TestCase):
         self.assertEqual(
             self.client.check_cert(self.certr), self.client.refresh(self.certr))
 
-    def test_fetch_chain(self):
+    def test_fetch_chain_no_up_link(self):
+        self.assertEqual([], self.client.fetch_chain(self.certr.update(
+            cert_chain_uri=None)))
+
+    def test_fetch_chain_single(self):
         # pylint: disable=protected-access
         self.client._get_cert = mock.MagicMock()
-        self.client._get_cert.return_value = ("response", "certificate")
-        self.assertEqual(self.client._get_cert(self.certr.cert_chain_uri)[1],
+        self.client._get_cert.return_value = (
+            mock.MagicMock(links={}), "certificate")
+        self.assertEqual([self.client._get_cert(self.certr.cert_chain_uri)[1]],
                          self.client.fetch_chain(self.certr))
 
-    def test_fetch_chain_no_up_link(self):
-        self.assertTrue(self.client.fetch_chain(self.certr.update(
-            cert_chain_uri=None)) is None)
+    def test_fetch_chain_max(self):
+        # pylint: disable=protected-access
+        up_response = mock.MagicMock(links={'up': {'url': 'http://cert'}})
+        noup_response = mock.MagicMock(links={})
+        self.client._get_cert = mock.MagicMock()
+        self.client._get_cert.side_effect = [
+            (up_response, "cert")] * 9 + [(noup_response, "last_cert")]
+        chain = self.client.fetch_chain(self.certr, max_length=10)
+        self.assertEqual(chain, ["cert"] * 9 + ["last_cert"])
+
+    def test_fetch_chain_too_many(self):  # recursive
+        # pylint: disable=protected-access
+        response = mock.MagicMock(links={'up': {'url': 'http://cert'}})
+        self.client._get_cert = mock.MagicMock()
+        self.client._get_cert.return_value = (response, "certificate")
+        self.assertRaises(errors.Error, self.client.fetch_chain, self.certr)
 
     def test_revoke(self):
         self.client.revoke(self.certr.body)
-        self.net.post.assert_called_once_with(messages.Revocation.url(
-            self.client.new_reg_uri), mock.ANY)
+        self.net.post.assert_called_once_with(
+            self.directory[messages.Revocation], mock.ANY, content_type=None)
 
     def test_revoke_bad_status_raises_error(self):
         self.response.status_code = http_client.METHOD_NOT_ALLOWED
@@ -375,11 +410,14 @@ class ClientNetworkTest(unittest.TestCase):
             # pylint: disable=missing-docstring
             def __init__(self, value):
                 self.value = value
+
             def to_partial_json(self):
                 return {'foo': self.value}
+
             @classmethod
             def from_json(cls, value):
                 pass  # pragma: no cover
+
         # pylint: disable=protected-access
         jws_dump = self.net._wrap_in_jws(
             MockJSONDeSerializable('foo'), nonce=b'Tg')
@@ -483,6 +521,7 @@ class ClientNetworkWithMockedResponseTest(unittest.TestCase):
 
         self.all_nonces = [jose.b64encode(b'Nonce'), jose.b64encode(b'Nonce2')]
         self.available_nonces = self.all_nonces[:]
+
         def send_request(*args, **kwargs):
             # pylint: disable=unused-argument,missing-docstring
             if self.available_nonces:

acme/acme/crypto_util.py

@@ -69,8 +69,8 @@ def _serve_sni(certs, sock, reuseaddr=True, method=_DEFAULT_DVSNI_SSL_METHOD,
                 raise errors.Error(error)
 
 
-def _probe_sni(name, host, port=443, timeout=300,
-               method=_DEFAULT_DVSNI_SSL_METHOD, source_address=('0', 0)):
+def probe_sni(name, host, port=443, timeout=300,
+              method=_DEFAULT_DVSNI_SSL_METHOD, source_address=('0', 0)):
     """Probe SNI server for SSL certificate.
 
     :param bytes name: Byte string to send as the server name in the
@@ -155,13 +155,18 @@ def _pyopenssl_cert_or_req_san(cert_or_req):
             for part in parts if part.startswith(prefix)]
 
 
-def gen_ss_cert(key, domains, not_before=None, validity=(7 * 24 * 60 * 60)):
+def gen_ss_cert(key, domains, not_before=None,
+                validity=(7 * 24 * 60 * 60), force_san=True):
     """Generate new self-signed certificate.
 
     :type domains: `list` of `unicode`
     :param OpenSSL.crypto.PKey key:
+    :param bool force_san:
 
-    Uses key and contains all domains.
+    If more than one domain is provided, all of the domains are put into
+    ``subjectAltName`` X.509 extension and first domain is set as the
+    subject CN. If only one domain is provided no ``subjectAltName``
+    extension is used, unless `force_san` is ``True``.
 
     """
     assert domains, "Must provide one or more hostnames for the cert."
@@ -178,7 +183,7 @@ def gen_ss_cert(key, domains, not_before=None, validity=(7 * 24 * 60 * 60)):
     # TODO: what to put into cert.get_subject()?
     cert.set_issuer(cert.get_subject())
 
-    if len(domains) > 1:
+    if force_san or len(domains) > 1:
         extensions.append(OpenSSL.crypto.X509Extension(
             b"subjectAltName",
             critical=False,

acme/acme/crypto_util_test.py

@@ -13,7 +13,7 @@ from acme import test_util
 
 
 class ServeProbeSNITest(unittest.TestCase):
-    """Tests for acme.crypto_util._serve_sni/_probe_sni."""
+    """Tests for acme.crypto_util._serve_sni/probe_sni."""
 
     def setUp(self):
         self.cert = test_util.load_cert('cert.pem')
@@ -45,8 +45,8 @@ class ServeProbeSNITest(unittest.TestCase):
         self.server.join()
 
     def _probe(self, name):
-        from acme.crypto_util import _probe_sni
-        return jose.ComparableX509(_probe_sni(
+        from acme.crypto_util import probe_sni
+        return jose.ComparableX509(probe_sni(
             name, host='127.0.0.1', port=self.port))
 
     def test_probe_ok(self):
@@ -55,10 +55,11 @@ class ServeProbeSNITest(unittest.TestCase):
     def test_probe_not_recognized_name(self):
         self.assertRaises(errors.Error, self._probe, b'bar')
 
-    def test_probe_connection_error(self):
-        self._probe(b'foo')
-        time.sleep(1)  # TODO: avoid race conditions in other way
-        self.assertRaises(errors.Error, self._probe, b'bar')
+    # TODO: py33/py34 tox hangs forever on do_hendshake in second probe
+    #def probe_connection_error(self):
+    #    self._probe(b'foo')
+    #    #time.sleep(1)  # TODO: avoid race conditions in other way
+    #    self.assertRaises(errors.Error, self._probe, b'bar')
 
 
 class PyOpenSSLCertOrReqSANTest(unittest.TestCase):

acme/acme/fields.py

@@ -1,9 +1,34 @@
 """ACME JSON fields."""
+import logging
+
 import pyrfc3339
 
 from acme import jose
 
 
+logger = logging.getLogger(__name__)
+
+
+class Fixed(jose.Field):
+    """Fixed field."""
+
+    def __init__(self, json_name, value):
+        self.value = value
+        super(Fixed, self).__init__(
+            json_name=json_name, default=value, omitempty=False)
+
+    def decode(self, value):
+        if value != self.value:
+            raise jose.DeserializationError('Expected {0!r}'.format(self.value))
+        return self.value
+
+    def encode(self, value):
+        if value != self.value:
+            logger.warn(
+                'Overriding fixed field (%s) with %r', self.json_name, value)
+        return value
+
+
 class RFC3339Field(jose.Field):
     """RFC3339 field encoder/decoder.
 
@@ -31,8 +56,6 @@ class Resource(jose.Field):
     def __init__(self, resource_type, *args, **kwargs):
         self.resource_type = resource_type
         super(Resource, self).__init__(
-            # TODO: omitempty used only to trick
-            # JSONObjectWithFieldsMeta._defaults..., server implementation
             'resource', default=resource_type, *args, **kwargs)
 
     def decode(self, value):

acme/acme/fields_test.py

@@ -7,6 +7,26 @@ import pytz
 from acme import jose
 
 
+class FixedTest(unittest.TestCase):
+    """Tests for acme.fields.Fixed."""
+
+    def setUp(self):
+        from acme.fields import Fixed
+        self.field = Fixed('name', 'x')
+
+    def test_decode(self):
+        self.assertEqual('x', self.field.decode('x'))
+
+    def test_decode_bad(self):
+        self.assertRaises(jose.DeserializationError, self.field.decode, 'y')
+
+    def test_encode(self):
+        self.assertEqual('x', self.field.encode('x'))
+
+    def test_encode_override(self):
+        self.assertEqual('y', self.field.encode('y'))
+
+
 class RFC3339FieldTest(unittest.TestCase):
     """Tests for acme.fields.RFC3339Field."""
 

acme/acme/jose/errors.py

@@ -8,6 +8,10 @@ class Error(Exception):
 class DeserializationError(Error):
     """JSON deserialization error."""
 
+    def __str__(self):
+        return "Deserialization error: {0}".format(
+            super(DeserializationError, self).__str__())
+
 
 class SerializationError(Error):
     """JSON serialization error."""

acme/acme/jose/interfaces.py

@@ -5,6 +5,7 @@ import json
 
 import six
 
+from acme.jose import errors
 from acme.jose import util
 
 # pylint: disable=no-self-argument,no-method-argument,no-init,inherit-non-class
@@ -40,7 +41,7 @@ class JSONDeSerializable(object):
         be encoded into a JSON document. **Full serialization** produces
         a Python object composed of only basic types as required by the
         :ref:`conversion table <conversion-table>`. **Partial
-        serialization** (acomplished by :meth:`to_partial_json`)
+        serialization** (accomplished by :meth:`to_partial_json`)
         produces a Python object that might also be built from other
         :class:`JSONDeSerializable` objects.
 
@@ -172,7 +173,11 @@ class JSONDeSerializable(object):
     @classmethod
     def json_loads(cls, json_string):
         """Deserialize from JSON document string."""
-        return cls.from_json(json.loads(json_string))
+        try:
+            loads = json.loads(json_string)
+        except ValueError as error:
+            raise errors.DeserializationError(error)
+        return cls.from_json(loads)
 
     def json_dumps(self, **kwargs):
         """Dump to JSON string using proper serializer.
@@ -189,7 +194,7 @@ class JSONDeSerializable(object):
         :rtype: str
 
         """
-        return self.json_dumps(sort_keys=True, indent=4, separators=(',', ': '))
+        return self.json_dumps(sort_keys=True, indent=4)
 
     @classmethod
     def json_dump_default(cls, python_object):

acme/acme/jose/interfaces_test.py

@@ -1,6 +1,8 @@
 """Tests for acme.jose.interfaces."""
 import unittest
 
+import six
+
 
 class JSONDeSerializableTest(unittest.TestCase):
     # pylint: disable=too-many-instance-attributes
@@ -90,8 +92,9 @@ class JSONDeSerializableTest(unittest.TestCase):
         self.assertEqual('["foo1", "foo2"]', self.seq.json_dumps())
 
     def test_json_dumps_pretty(self):
-        self.assertEqual(
-            self.seq.json_dumps_pretty(), '[\n    "foo1",\n    "foo2"\n]')
+        filler = ' ' if six.PY2 else ''
+        self.assertEqual(self.seq.json_dumps_pretty(),
+                         '[\n    "foo1",{0}\n    "foo2"\n]'.format(filler))
 
     def test_json_dump_default(self):
         from acme.jose.interfaces import JSONDeSerializable

acme/acme/jose/json_util.py

@@ -221,6 +221,22 @@ class JSONObjectWithFields(util.ImmutableMap, interfaces.JSONDeSerializable):
         super(JSONObjectWithFields, self).__init__(
             **(dict(self._defaults(), **kwargs)))
 
+    def encode(self, name):
+        """Encode a single field.
+
+        :param str name: Name of the field to be encoded.
+
+        :raises erors.SerializationError: if field cannot be serialized
+        :raises errors.Error: if field could not be found
+
+        """
+        try:
+            field = self._fields[name]
+        except KeyError:
+            raise errors.Error("Field not found: {0}".format(name))
+
+        return field.encode(getattr(self, name))
+
     def fields_to_partial_json(self):
         """Serialize fields to JSON."""
         jobj = {}
@@ -291,6 +307,7 @@ def encode_b64jose(data):
     # b64encode produces ASCII characters only
     return b64.b64encode(data).decode('ascii')
 
+
 def decode_b64jose(data, size=None, minimum=False):
     """Decode JOSE Base-64 field.
 
@@ -308,12 +325,14 @@ def decode_b64jose(data, size=None, minimum=False):
     except error_cls as error:
         raise errors.DeserializationError(error)
 
-    if size is not None and ((not minimum and len(decoded) != size)
-                             or (minimum and len(decoded) < size)):
-        raise errors.DeserializationError()
+    if size is not None and ((not minimum and len(decoded) != size) or
+                             (minimum and len(decoded) < size)):
+        raise errors.DeserializationError(
+            "Expected at least or exactly {0} bytes".format(size))
 
     return decoded
 
+
 def encode_hex16(value):
     """Hexlify.
 
@@ -323,6 +342,7 @@ def encode_hex16(value):
     """
     return binascii.hexlify(value).decode()
 
+
 def decode_hex16(value, size=None, minimum=False):
     """Decode hexlified field.
 
@@ -335,8 +355,8 @@ def decode_hex16(value, size=None, minimum=False):
 
     """
     value = value.encode()
-    if size is not None and ((not minimum and len(value) != size * 2)
-                             or (minimum and len(value) < size * 2)):
+    if size is not None and ((not minimum and len(value) != size * 2) or
+                             (minimum and len(value) < size * 2)):
         raise errors.DeserializationError()
     error_cls = TypeError if six.PY2 else binascii.Error
     try:
@@ -344,6 +364,7 @@ def decode_hex16(value, size=None, minimum=False):
     except error_cls as error:
         raise errors.DeserializationError(error)
 
+
 def encode_cert(cert):
     """Encode certificate as JOSE Base-64 DER.
 
@@ -354,6 +375,7 @@ def encode_cert(cert):
     return encode_b64jose(OpenSSL.crypto.dump_certificate(
         OpenSSL.crypto.FILETYPE_ASN1, cert))
 
+
 def decode_cert(b64der):
     """Decode JOSE Base-64 DER-encoded certificate.
 
@@ -367,6 +389,7 @@ def decode_cert(b64der):
     except OpenSSL.crypto.Error as error:
         raise errors.DeserializationError(error)
 
+
 def encode_csr(csr):
     """Encode CSR as JOSE Base-64 DER.
 
@@ -377,6 +400,7 @@ def encode_csr(csr):
     return encode_b64jose(OpenSSL.crypto.dump_certificate_request(
         OpenSSL.crypto.FILETYPE_ASN1, csr))
 
+
 def decode_csr(b64der):
     """Decode JOSE Base-64 DER-encoded CSR.
 
@@ -418,7 +442,9 @@ class TypedJSONObjectWithFields(JSONObjectWithFields):
     def get_type_cls(cls, jobj):
         """Get the registered class for ``jobj``."""
         if cls in six.itervalues(cls.TYPES):
-            assert jobj[cls.type_field_name]
+            if cls.type_field_name not in jobj:
+                raise errors.DeserializationError(
+                    "Missing type field ({0})".format(cls.type_field_name))
             # cls is already registered type_cls, force to use it
             # so that, e.g Revocation.from_json(jobj) fails if
             # jobj["type"] != "revocation".

acme/acme/jose/json_util_test.py

@@ -52,6 +52,7 @@ class FieldTest(unittest.TestCase):
             # pylint: disable=missing-docstring
             def to_partial_json(self):
                 return 'foo'  # pragma: no cover
+
             @classmethod
             def from_json(cls, jobj):
                 pass  # pragma: no cover
@@ -93,14 +94,18 @@ class JSONObjectWithFieldsMetaTest(unittest.TestCase):
         self.field2 = Field('Baz2')
         # pylint: disable=invalid-name,missing-docstring,too-few-public-methods
         # pylint: disable=blacklisted-name
+
         @six.add_metaclass(JSONObjectWithFieldsMeta)
         class A(object):
             __slots__ = ('bar',)
             baz = self.field
+
         class B(A):
             pass
+
         class C(A):
             baz = self.field2
+
         self.a_cls = A
         self.b_cls = B
         self.c_cls = C
@@ -160,6 +165,18 @@ class JSONObjectWithFieldsTest(unittest.TestCase):
     def test_init_defaults(self):
         self.assertEqual(self.mock, self.MockJSONObjectWithFields(y=2, z=3))
 
+    def test_encode(self):
+        self.assertEqual(10, self.MockJSONObjectWithFields(
+            x=5, y=0, z=0).encode("x"))
+
+    def test_encode_wrong_field(self):
+        self.assertRaises(errors.Error, self.mock.encode, 'foo')
+
+    def test_encode_serialization_error_passthrough(self):
+        self.assertRaises(
+            errors.SerializationError,
+            self.MockJSONObjectWithFields(y=500, z=None).encode, "y")
+
     def test_fields_to_partial_json_omits_empty(self):
         self.assertEqual(self.mock.fields_to_partial_json(), {'y': 2, 'Z': 3})
 

acme/acme/jose/jwa.py

@@ -21,7 +21,7 @@ from acme.jose import jwk
 logger = logging.getLogger(__name__)
 
 
-class JWA(interfaces.JSONDeSerializable): # pylint: disable=abstract-method
+class JWA(interfaces.JSONDeSerializable):  # pylint: disable=abstract-method
     # pylint: disable=too-few-public-methods
     # for some reason disable=abstract-method has to be on the line
     # above...
@@ -159,7 +159,7 @@ class _JWAES(JWASignature):  # pylint: disable=abstract-class-not-used
     def sign(self, key, msg):  # pragma: no cover
         raise NotImplementedError()
 
-    def verify(self, key, msg, sig): # pragma: no cover
+    def verify(self, key, msg, sig):  # pragma: no cover
         raise NotImplementedError()
 
 

acme/acme/jose/jwk.py

@@ -1,10 +1,12 @@
 """JSON Web Key."""
 import abc
 import binascii
+import json
 import logging
 
 import cryptography.exceptions
 from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import ec
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -27,6 +29,32 @@ class JWK(json_util.TypedJSONObjectWithFields):
     cryptography_key_types = ()
     """Subclasses should override."""
 
+    required = NotImplemented
+    """Required members of public key's representation as defined by JWK/JWA."""
+
+    _thumbprint_json_dumps_params = {
+        # "no whitespace or line breaks before or after any syntactic
+        # elements"
+        'indent': 0,
+        'separators': (',', ':'),
+        # "members ordered lexicographically by the Unicode [UNICODE]
+        # code points of the member names"
+        'sort_keys': True,
+    }
+
+    def thumbprint(self, hash_function=hashes.SHA256):
+        """Compute JWK Thumbprint.
+
+        https://tools.ietf.org/html/rfc7638
+
+        """
+        digest = hashes.Hash(hash_function(), backend=default_backend())
+        digest.update(json.dumps(
+            dict((k, v) for k, v in six.iteritems(self.to_json())
+                 if k in self.required),
+            **self._thumbprint_json_dumps_params).encode())
+        return digest.finalize()
+
     @abc.abstractmethod
     def public_key(self):  # pragma: no cover
         """Generate JWK with public key.
@@ -60,7 +88,7 @@ class JWK(json_util.TypedJSONObjectWithFields):
                 exceptions[loader] = error
 
         # no luck
-        raise errors.Error("Unable to deserialize key: {0}".format(exceptions))
+        raise errors.Error('Unable to deserialize key: {0}'.format(exceptions))
 
     @classmethod
     def load(cls, data, password=None, backend=None):
@@ -81,17 +109,17 @@ class JWK(json_util.TypedJSONObjectWithFields):
         try:
             key = cls._load_cryptography_key(data, password, backend)
         except errors.Error as error:
-            logger.debug("Loading symmetric key, assymentric failed: %s", error)
+            logger.debug('Loading symmetric key, assymentric failed: %s', error)
             return JWKOct(key=data)
 
         if cls.typ is not NotImplemented and not isinstance(
                 key, cls.cryptography_key_types):
-            raise errors.Error("Unable to deserialize {0} into {1}".format(
+            raise errors.Error('Unable to deserialize {0} into {1}'.format(
                 key.__class__, cls.__class__))
         for jwk_cls in six.itervalues(cls.TYPES):
             if isinstance(key, jwk_cls.cryptography_key_types):
                 return jwk_cls(key=key)
-        raise errors.Error("Unsupported algorithm: {0}".format(key.__class__))
+        raise errors.Error('Unsupported algorithm: {0}'.format(key.__class__))
 
 
 @JWK.register
@@ -105,6 +133,7 @@ class JWKES(JWK):  # pragma: no cover
     typ = 'ES'
     cryptography_key_types = (
         ec.EllipticCurvePublicKey, ec.EllipticCurvePrivateKey)
+    required = ('crv', JWK.type_field_name, 'x', 'y')
 
     def fields_to_partial_json(self):
         raise NotImplementedError()
@@ -122,6 +151,7 @@ class JWKOct(JWK):
     """Symmetric JWK."""
     typ = 'oct'
     __slots__ = ('key',)
+    required = ('k', JWK.type_field_name)
 
     def fields_to_partial_json(self):
         # TODO: An "alg" member SHOULD also be present to identify the
@@ -150,6 +180,7 @@ class JWKRSA(JWK):
     typ = 'RSA'
     cryptography_key_types = (rsa.RSAPublicKey, rsa.RSAPrivateKey)
     __slots__ = ('key',)
+    required = ('e', JWK.type_field_name, 'n')
 
     def __init__(self, *args, **kwargs):
         if 'key' in kwargs and not isinstance(
@@ -204,7 +235,7 @@ class JWKRSA(JWK):
                     jobj.get(x) for x in ('p', 'q', 'dp', 'dq', 'qi'))
                 if tuple(param for param in all_params if param is None):
                     raise errors.Error(
-                        "Some private parameters are missing: {0}".format(
+                        'Some private parameters are missing: {0}'.format(
                             all_params))
                 p, q, dp, dq, qi = tuple(
                     cls._decode_param(x) for x in all_params)
@@ -231,7 +262,7 @@ class JWKRSA(JWK):
                 'n': numbers.n,
                 'e': numbers.e,
             }
-        else: # rsa.RSAPrivateKey
+        else:  # rsa.RSAPrivateKey
             private = self.key.private_numbers()
             public = self.key.public_key().public_numbers()
             params = {

acme/acme/jose/jwk_test.py

@@ -25,9 +25,24 @@ class JWKTest(unittest.TestCase):
         self.assertRaises(errors.Error, JWKRSA.load, DSA_PEM)
 
 
-class JWKOctTest(unittest.TestCase):
+class JWKTestBaseMixin(object):
+    """Mixin test for JWK subclass tests."""
+
+    thumbprint = NotImplemented
+
+    def test_thumbprint_private(self):
+        self.assertEqual(self.thumbprint, self.jwk.thumbprint())
+
+    def test_thumbprint_public(self):
+        self.assertEqual(self.thumbprint, self.jwk.public_key().thumbprint())
+
+
+class JWKOctTest(unittest.TestCase, JWKTestBaseMixin):
     """Tests for acme.jose.jwk.JWKOct."""
 
+    thumbprint = (b"=,\xdd;I\x1a+i\x02x\x8a\x12?06IM\xc2\x80"
+                  b"\xe4\xc3\x1a\xfc\x89\xf3)'\xce\xccm\xfd5")
+
     def setUp(self):
         from acme.jose.jwk import JWKOct
         self.jwk = JWKOct(key=b'foo')
@@ -52,10 +67,13 @@ class JWKOctTest(unittest.TestCase):
         self.assertTrue(self.jwk.public_key() is self.jwk)
 
 
-class JWKRSATest(unittest.TestCase):
+class JWKRSATest(unittest.TestCase, JWKTestBaseMixin):
     """Tests for acme.jose.jwk.JWKRSA."""
     # pylint: disable=too-many-instance-attributes
 
+    thumbprint = (b'\x08\xfa1\x87\x1d\x9b6H/*\x1eW\xc2\xe3\xf6P'
+                  b'\xefs\x0cKB\x87\xcf\x85yO\x045\x0e\x91\x80\x0b')
+
     def setUp(self):
         from acme.jose.jwk import JWKRSA
         self.jwk256 = JWKRSA(key=RSA256_KEY.public_key())
@@ -87,6 +105,7 @@ class JWKRSATest(unittest.TestCase):
             'dq': 'bHh2u7etM8LKKCF2pY2UdQ',
             'qi': 'oi45cEkbVoJjAbnQpFY87Q',
         })
+        self.jwk = self.private
 
     def test_init_auto_comparable(self):
         self.assertTrue(isinstance(

acme/acme/jose/jws.py

@@ -53,8 +53,8 @@ class Header(json_util.JSONObjectWithFields):
     .. warning:: This class does not support any extensions through
         the "crit" (Critical) Header Parameter (4.1.11) and as a
         conforming implementation, :meth:`from_json` treats its
-        occurence as an error. Please subclass if you seek for
-        a diferent behaviour.
+        occurrence as an error. Please subclass if you seek for
+        a different behaviour.
 
     :ivar x5tS256: "x5t#S256"
     :ivar str typ: MIME Media Type, inc. :const:`MediaType.PREFIX`.
@@ -294,10 +294,10 @@ class JWS(json_util.JSONObjectWithFields):
         # ... it must be in protected
 
         return (
-            b64.b64encode(self.signature.protected.encode('utf-8'))
-            + b'.' +
-            b64.b64encode(self.payload)
-            + b'.' +
+            b64.b64encode(self.signature.protected.encode('utf-8')) +
+            b'.' +
+            b64.b64encode(self.payload) +
+            b'.' +
             b64.b64encode(self.signature.signature))
 
     @classmethod
@@ -345,6 +345,7 @@ class JWS(json_util.JSONObjectWithFields):
                        signatures=tuple(cls.signature_cls.from_json(sig)
                                         for sig in jobj['signatures']))
 
+
 class CLI(object):
     """JWS CLI."""
 

acme/acme/jose/util.py

@@ -107,8 +107,8 @@ class ComparableRSAKey(ComparableKey):  # pylint: disable=too-few-public-methods
     """Wrapper for `cryptography` RSA keys.
 
     Wraps around:
-    - `cryptography.hazmat.primitives.assymetric.RSAPrivateKey`
-    - `cryptography.hazmat.primitives.assymetric.RSAPublicKey`
+    - `cryptography.hazmat.primitives.asymmetric.RSAPrivateKey`
+    - `cryptography.hazmat.primitives.asymmetric.RSAPublicKey`
 
     """
 

acme/acme/messages.py

@@ -1,11 +1,10 @@
 """ACME protocol messages."""
 import collections
 
-from six.moves.urllib import parse as urllib_parse  # pylint: disable=import-error
-
 from acme import challenges
 from acme import fields
 from acme import jose
+from acme import util
 
 
 class Error(jose.JSONObjectWithFields, Exception):
@@ -25,6 +24,7 @@ class Error(jose.JSONObjectWithFields, Exception):
         'connection': 'The server could not connect to the client for DV',
         'dnssec': 'The server could not validate a DNSSEC signed domain',
         'malformed': 'The request message was malformed',
+        'rateLimited': 'There were too many requests of a given type',
         'serverInternal': 'The server experienced an internal error',
         'tls': 'The server experienced a TLS error during DV',
         'unauthorized': 'The client lacks sufficient authorization',
@@ -128,6 +128,56 @@ class Identifier(jose.JSONObjectWithFields):
     value = jose.Field('value')
 
 
+class Directory(jose.JSONDeSerializable):
+    """Directory."""
+
+    _REGISTERED_TYPES = {}
+
+    @classmethod
+    def _canon_key(cls, key):
+        return getattr(key, 'resource_type', key)
+
+    @classmethod
+    def register(cls, resource_body_cls):
+        """Register resource."""
+        assert resource_body_cls.resource_type not in cls._REGISTERED_TYPES
+        cls._REGISTERED_TYPES[resource_body_cls.resource_type] = resource_body_cls
+        return resource_body_cls
+
+    def __init__(self, jobj):
+        canon_jobj = util.map_keys(jobj, self._canon_key)
+        if not set(canon_jobj).issubset(self._REGISTERED_TYPES):
+            # TODO: acme-spec is not clear about this: 'It is a JSON
+            # dictionary, whose keys are the "resource" values listed
+            # in {{https-requests}}'z
+            raise ValueError('Wrong directory fields')
+        # TODO: check that everything is an absolute URL; acme-spec is
+        # not clear on that
+        self._jobj = canon_jobj
+
+    def __getattr__(self, name):
+        try:
+            return self[name.replace('_', '-')]
+        except KeyError as error:
+            raise AttributeError(str(error))
+
+    def __getitem__(self, name):
+        try:
+            return self._jobj[self._canon_key(name)]
+        except KeyError:
+            raise KeyError('Directory field not found')
+
+    def to_partial_json(self):
+        return self._jobj
+
+    @classmethod
+    def from_json(cls, jobj):
+        try:
+            return cls(jobj)
+        except ValueError as error:
+            raise jose.DeserializationError(str(error))
+
+
 class Resource(jose.JSONObjectWithFields):
     """ACME Resource.
 
@@ -156,16 +206,36 @@ class Registration(ResourceBody):
     :ivar acme.jose.jwk.JWK key: Public key.
     :ivar tuple contact: Contact information following ACME spec,
         `tuple` of `unicode`.
-    :ivar unicode recovery_token:
     :ivar unicode agreement:
+    :ivar unicode authorizations: URI where
+        `messages.Registration.Authorizations` can be found.
+    :ivar unicode certificates: URI where
+        `messages.Registration.Certificates` can be found.
 
     """
     # on new-reg key server ignores 'key' and populates it based on
     # JWS.signature.combined.jwk
     key = jose.Field('key', omitempty=True, decoder=jose.JWK.from_json)
     contact = jose.Field('contact', omitempty=True, default=())
-    recovery_token = jose.Field('recoveryToken', omitempty=True)
     agreement = jose.Field('agreement', omitempty=True)
+    authorizations = jose.Field('authorizations', omitempty=True)
+    certificates = jose.Field('certificates', omitempty=True)
+
+    class Authorizations(jose.JSONObjectWithFields):
+        """Authorizations granted to Account in the process of registration.
+
+        :ivar tuple authorizations: URIs to Authorization Resources.
+
+        """
+        authorizations = jose.Field('authorizations')
+
+    class Certificates(jose.JSONObjectWithFields):
+        """Certificates granted to Account in the process of registration.
+
+        :ivar tuple certificates: URIs to Certificate Resources.
+
+        """
+        certificates = jose.Field('certificates')
 
     phone_prefix = 'tel:'
     email_prefix = 'mailto:'
@@ -196,16 +266,20 @@ class Registration(ResourceBody):
         """All emails found in the ``contact`` field."""
         return self._filter_contact(self.email_prefix)
 
+
+@Directory.register
 class NewRegistration(Registration):
     """New registration."""
     resource_type = 'new-reg'
     resource = fields.Resource(resource_type)
 
+
 class UpdateRegistration(Registration):
     """Update registration."""
     resource_type = 'reg'
     resource = fields.Resource(resource_type)
 
+
 class RegistrationResource(ResourceWithURI):
     """Registration Resource.
 
@@ -233,7 +307,7 @@ class ChallengeBody(ResourceBody):
         call ``challb.x`` to get ``challb.chall.x`` contents.
     :ivar acme.messages.Status status:
     :ivar datetime.datetime validated:
-    :ivar Error error:
+    :ivar messages.Error error:
 
     """
     __slots__ = ('chall',)
@@ -308,11 +382,14 @@ class Authorization(ResourceBody):
         return tuple(tuple(self.challenges[idx] for idx in combo)
                      for combo in self.combinations)
 
+
+@Directory.register
 class NewAuthorization(Authorization):
     """New authorization."""
     resource_type = 'new-authz'
     resource = fields.Resource(resource_type)
 
+
 class AuthorizationResource(ResourceWithURI):
     """Authorization Resource.
 
@@ -324,18 +401,17 @@ class AuthorizationResource(ResourceWithURI):
     new_cert_uri = jose.Field('new_cert_uri')
 
 
+@Directory.register
 class CertificateRequest(jose.JSONObjectWithFields):
     """ACME new-cert request.
 
     :ivar acme.jose.util.ComparableX509 csr:
         `OpenSSL.crypto.X509Req` wrapped in `.ComparableX509`
-    :ivar tuple authorizations: `tuple` of URIs (`str`)
 
     """
     resource_type = 'new-cert'
     resource = fields.Resource(resource_type)
     csr = jose.Field('csr', decoder=jose.decode_csr, encoder=jose.encode_csr)
-    authorizations = jose.Field('authorizations', decoder=tuple)
 
 
 class CertificateResource(ResourceWithURI):
@@ -351,6 +427,7 @@ class CertificateResource(ResourceWithURI):
     authzrs = jose.Field('authzrs')
 
 
+@Directory.register
 class Revocation(jose.JSONObjectWithFields):
     """Revocation message.
 
@@ -362,16 +439,3 @@ class Revocation(jose.JSONObjectWithFields):
     resource = fields.Resource(resource_type)
     certificate = jose.Field(
         'certificate', decoder=jose.decode_cert, encoder=jose.encode_cert)
-
-    # TODO: acme-spec#138, this allows only one ACME server instance per domain
-    PATH = '/acme/revoke-cert'
-    """Path to revocation URL, see `url`"""
-
-    @classmethod
-    def url(cls, base):
-        """Get revocation URL.
-
-        :param str base: New Registration Resource or server (root) URL.
-
-        """
-        return urllib_parse.urljoin(base, cls.PATH)

acme/acme/messages_test.py

@@ -60,6 +60,7 @@ class ConstantTest(unittest.TestCase):
 
     def setUp(self):
         from acme.messages import _Constant
+
         class MockConstant(_Constant):  # pylint: disable=missing-docstring
             POSSIBLE_NAMES = {}
 
@@ -92,6 +93,45 @@ class ConstantTest(unittest.TestCase):
         self.assertFalse(self.const_a != const_a_prime)
 
 
+class DirectoryTest(unittest.TestCase):
+    """Tests for acme.messages.Directory."""
+
+    def setUp(self):
+        from acme.messages import Directory
+        self.dir = Directory({
+            'new-reg': 'reg',
+            mock.MagicMock(resource_type='new-cert'): 'cert',
+        })
+
+    def test_init_wrong_key_value_error(self):
+        from acme.messages import Directory
+        self.assertRaises(ValueError, Directory, {'foo': 'bar'})
+
+    def test_getitem(self):
+        self.assertEqual('reg', self.dir['new-reg'])
+        from acme.messages import NewRegistration
+        self.assertEqual('reg', self.dir[NewRegistration])
+        self.assertEqual('reg', self.dir[NewRegistration()])
+
+    def test_getitem_fails_with_key_error(self):
+        self.assertRaises(KeyError, self.dir.__getitem__, 'foo')
+
+    def test_getattr(self):
+        self.assertEqual('reg', self.dir.new_reg)
+
+    def test_getattr_fails_with_attribute_error(self):
+        self.assertRaises(AttributeError, self.dir.__getattr__, 'foo')
+
+    def test_to_partial_json(self):
+        self.assertEqual(
+            self.dir.to_partial_json(), {'new-reg': 'reg', 'new-cert': 'cert'})
+
+    def test_from_json_deserialization_error_on_wrong_key(self):
+        from acme.messages import Directory
+        self.assertRaises(
+            jose.DeserializationError, Directory.from_json, {'foo': 'bar'})
+
+
 class RegistrationTest(unittest.TestCase):
     """Tests for acme.messages.Registration."""
 
@@ -101,18 +141,15 @@ class RegistrationTest(unittest.TestCase):
             'mailto:admin@foo.com',
             'tel:1234',
         )
-        recovery_token = 'XYZ'
         agreement = 'https://letsencrypt.org/terms'
 
         from acme.messages import Registration
-        self.reg = Registration(
-            key=key, contact=contact, recovery_token=recovery_token,
-            agreement=agreement)
-        self.reg_none = Registration()
+        self.reg = Registration(key=key, contact=contact, agreement=agreement)
+        self.reg_none = Registration(authorizations='uri/authorizations',
+                                     certificates='uri/certificates')
 
         self.jobj_to = {
             'contact': contact,
-            'recoveryToken': recovery_token,
             'agreement': agreement,
             'key': key,
         }
@@ -145,6 +182,17 @@ class RegistrationTest(unittest.TestCase):
         hash(Registration.from_json(self.jobj_from))
 
 
+class UpdateRegistrationTest(unittest.TestCase):
+    """Tests for acme.messages.UpdateRegistration."""
+
+    def test_empty(self):
+        from acme.messages import UpdateRegistration
+        jstring = '{"resource": "reg"}'
+        self.assertEqual(jstring, UpdateRegistration().json_dumps())
+        self.assertEqual(
+            UpdateRegistration(), UpdateRegistration.json_loads(jstring))
+
+
 class RegistrationResourceTest(unittest.TestCase):
     """Tests for acme.messages.RegistrationResource."""
 
@@ -177,7 +225,8 @@ class ChallengeBodyTest(unittest.TestCase):
     """Tests for acme.messages.ChallengeBody."""
 
     def setUp(self):
-        self.chall = challenges.DNS(token='foo')
+        self.chall = challenges.DNS(token=jose.b64decode(
+            'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'))
 
         from acme.messages import ChallengeBody
         from acme.messages import Error
@@ -193,7 +242,7 @@ class ChallengeBodyTest(unittest.TestCase):
             'uri': 'http://challb',
             'status': self.status,
             'type': 'dns',
-            'token': 'foo',
+            'token': 'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA',
             'error': error,
         }
         self.jobj_from = self.jobj_to.copy()
@@ -203,7 +252,6 @@ class ChallengeBodyTest(unittest.TestCase):
             'detail': 'Unable to communicate with DNS server',
         }
 
-
     def test_to_partial_json(self):
         self.assertEqual(self.jobj_to, self.challb.to_partial_json())
 
@@ -216,7 +264,8 @@ class ChallengeBodyTest(unittest.TestCase):
         hash(ChallengeBody.from_json(self.jobj_from))
 
     def test_proxy(self):
-        self.assertEqual('foo', self.challb.token)
+        self.assertEqual(jose.b64decode(
+            'evaGxfADs6pSRb2LAv9IZf17Dt3juxGJ-PCt92wr-oA'), self.challb.token)
 
 
 class AuthorizationTest(unittest.TestCase):
@@ -225,14 +274,16 @@ class AuthorizationTest(unittest.TestCase):
     def setUp(self):
         from acme.messages import ChallengeBody
         from acme.messages import STATUS_VALID
+
         self.challbs = (
             ChallengeBody(
                 uri='http://challb1', status=STATUS_VALID,
-                chall=challenges.SimpleHTTP(token='IlirfxKKXAsHtmzK29Pj8A')),
+                chall=challenges.SimpleHTTP(token=b'IlirfxKKXAsHtmzK29Pj8A')),
             ChallengeBody(uri='http://challb2', status=STATUS_VALID,
-                          chall=challenges.DNS(token='DGyRejmCefe7v4NfDGDKfA')),
+                          chall=challenges.DNS(
+                              token=b'DGyRejmCefe7v4NfDGDKfA')),
             ChallengeBody(uri='http://challb3', status=STATUS_VALID,
-                          chall=challenges.RecoveryToken()),
+                          chall=challenges.RecoveryContact()),
         )
         combinations = ((0, 2), (1, 2))
 
@@ -283,7 +334,7 @@ class CertificateRequestTest(unittest.TestCase):
 
     def setUp(self):
         from acme.messages import CertificateRequest
-        self.req = CertificateRequest(csr=CSR, authorizations=('foo',))
+        self.req = CertificateRequest(csr=CSR)
 
     def test_json_de_serializable(self):
         self.assertTrue(isinstance(self.req, jose.JSONDeSerializable))
@@ -311,13 +362,6 @@ class CertificateResourceTest(unittest.TestCase):
 class RevocationTest(unittest.TestCase):
     """Tests for acme.messages.RevocationTest."""
 
-    def test_url(self):
-        from acme.messages import Revocation
-        url = 'https://letsencrypt-demo.org/acme/revoke-cert'
-        self.assertEqual(url, Revocation.url('https://letsencrypt-demo.org'))
-        self.assertEqual(
-            url, Revocation.url('https://letsencrypt-demo.org/acme/new-reg'))
-
     def setUp(self):
         from acme.messages import Revocation
         self.rev = Revocation(certificate=CERT)

acme/acme/other.py

@@ -36,7 +36,7 @@ class Signature(jose.JSONObjectWithFields):
         :param bytes msg: Message to be signed.
 
         :param key: Key used for signing.
-        :type key: `cryptography.hazmat.primitives.assymetric.rsa.RSAPrivateKey`
+        :type key: `cryptography.hazmat.primitives.asymmetric.rsa.RSAPrivateKey`
             (optionally wrapped in `.ComparableRSAKey`).
 
         :param bytes nonce: Nonce to be used. If None, nonce of

acme/acme/test_util.py

@@ -1,5 +1,3 @@
-# Symlinked in letsencrypt/tests/test_util.py, casues duplicate-code
-# warning that cannot be disabled locally.
 """Test utilities.
 
 .. warning:: This module is not part of the public API.
@@ -20,12 +18,14 @@ def vector_path(*names):
     return pkg_resources.resource_filename(
         __name__, os.path.join('testdata', *names))
 
+
 def load_vector(*names):
     """Load contents of a test vector."""
     # luckily, resource_string opens file in binary mode
     return pkg_resources.resource_string(
         __name__, os.path.join('testdata', *names))
 
+
 def _guess_loader(filename, loader_pem, loader_der):
     _, ext = os.path.splitext(filename)
     if ext.lower() == '.pem':
@@ -35,6 +35,7 @@ def _guess_loader(filename, loader_pem, loader_der):
     else:  # pragma: no cover
         raise ValueError("Loader could not be recognized based on extension")
 
+
 def load_cert(*names):
     """Load certificate."""
     loader = _guess_loader(
@@ -42,6 +43,7 @@ def load_cert(*names):
     return jose.ComparableX509(OpenSSL.crypto.load_certificate(
         loader, load_vector(*names)))
 
+
 def load_csr(*names):
     """Load certificate request."""
     loader = _guess_loader(
@@ -49,6 +51,7 @@ def load_csr(*names):
     return jose.ComparableX509(OpenSSL.crypto.load_certificate_request(
         loader, load_vector(*names)))
 
+
 def load_rsa_private_key(*names):
     """Load RSA private key."""
     loader = _guess_loader(names[-1], serialization.load_pem_private_key,
@@ -56,6 +59,7 @@ def load_rsa_private_key(*names):
     return jose.ComparableRSAKey(loader(
         load_vector(*names), password=None, backend=default_backend()))
 
+
 def load_pyopenssl_private_key(*names):
     """Load pyOpenSSL private key."""
     loader = _guess_loader(

acme/acme/util.py

@@ -0,0 +1,7 @@
+"""ACME utilities."""
+import six
+
+
+def map_keys(dikt, func):
+    """Map dictionary keys."""
+    return dict((func(key), value) for key, value in six.iteritems(dikt))

acme/acme/util_test.py

@@ -0,0 +1,16 @@
+"""Tests for acme.util."""
+import unittest
+
+
+class MapKeysTest(unittest.TestCase):
+    """Tests for acme.util.map_keys."""
+
+    def test_it(self):
+        from acme.util import map_keys
+        self.assertEqual({'a': 'b', 'c': 'd'},
+                         map_keys({'a': 'b', 'c': 'd'}, lambda key: key))
+        self.assertEqual({2: 2, 4: 4}, map_keys({1: 2, 3: 4}, lambda x: x + 1))
+
+
+if __name__ == '__main__':
+    unittest.main()  # pragma: no cover

acme/setup.py

@@ -4,27 +4,33 @@ from setuptools import setup
 from setuptools import find_packages
 
 
+version = '0.1.0.dev0'
+
 install_requires = [
-    'argparse',
     # load_pem_private/public_key (>=0.6)
     # rsa_recover_prime_factors (>=0.8)
     'cryptography>=0.8',
-    'mock<1.1.0',  # py26
-    'pyrfc3339',
     'ndg-httpsclient',  # urllib3 InsecurePlatformWarning (#304)
     'pyasn1',  # urllib3 InsecurePlatformWarning (#304)
     # Connection.set_tlsext_host_name (>=0.13), X509Req.get_extensions (>=0.15)
     'PyOpenSSL>=0.15',
+    'pyrfc3339',
     'pytz',
     'requests',
+    'setuptools',  # pkg_resources
     'six',
     'werkzeug',
 ]
 
 # env markers in extras_require cause problems with older pip: #517
 if sys.version_info < (2, 7):
-    # only some distros recognize stdlib argparse as already satisfying
-    install_requires.append('argparse')
+    install_requires.extend([
+        # only some distros recognize stdlib argparse as already satisfying
+        'argparse',
+        'mock<1.1.0',
+    ])
+else:
+    install_requires.append('mock')
 
 testing_extras = [
     'nose',
@@ -34,7 +40,25 @@ testing_extras = [
 
 setup(
     name='acme',
+    version=version,
+    description='ACME protocol implementation',
+    url='https://github.com/letsencrypt/letsencrypt',
+    author="Let's Encrypt Project",
+    author_email='client-dev@letsencrypt.org',
+    license='Apache License 2.0',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: Apache Software License',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Topic :: Internet :: WWW/HTTP',
+        'Topic :: Security',
+    ],
+
     packages=find_packages(),
+    include_package_data=True,
     install_requires=install_requires,
     extras_require={
         'testing': testing_extras,

bootstrap/_rpm_common.sh

@@ -4,8 +4,19 @@
 #   - Fedora 22 (x64)
 #   - Centos 7 (x64: on AWS EC2 t2.micro, DigitalOcean droplet)
 
+if type yum 2>/dev/null
+then
+  tool=yum
+elif type dnf 2>/dev/null
+then
+  tool=dnf
+else
+  echo "Neither yum nor dnf found. Aborting bootstrap!"
+  exit 1
+fi
+
 # "git-core" seems to be an alias for "git" in CentOS 7 (yum search fails)
-yum install -y \
+$tool install -y \
   git-core \
   python \
   python-devel \

bootstrap/archlinux.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+# "python-virtualenv" is Python3, but "python2-virtualenv" provides
+# only "virtualenv2" binary, not "virtualenv" necessary in
+# ./bootstrap/dev/_common_venv.sh
+pacman -S \
+  git \
+  python2 \
+  python-virtualenv \
+  gcc \
+  dialog \
+  augeas \
+  openssl \
+  libffi \
+  ca-certificates \

bootstrap/dev/README

@@ -0,0 +1 @@
+This directory contains developer setup.

bootstrap/dev/_venv_common.sh

@@ -0,0 +1,25 @@
+#!/bin/sh -xe
+
+VENV_NAME=${VENV_NAME:-venv}
+
+# .egg-info directories tend to cause bizzaire problems (e.g. `pip -e
+# .` might unexpectedly install letshelp-letsencrypt only, in case
+# `python letshelp-letsencrypt/setup.py build` has been called
+# earlier)
+rm -rf *.egg-info
+
+# virtualenv setup is NOT idempotent: shutil.Error:
+# `/home/jakub/dev/letsencrypt/letsencrypt/venv/bin/python2` and
+# `venv/bin/python2` are the same file
+mv $VENV_NAME "$VENV_NAME.$(date +%s).bak" || true
+virtualenv --no-site-packages $VENV_NAME $VENV_ARGS
+. ./$VENV_NAME/bin/activate
+
+# Separately install setuptools and pip to make sure following
+# invocations use latest
+pip install -U setuptools
+pip install -U pip
+pip install "$@"
+
+echo "Please run the following command to activate developer environment:"
+echo "source $VENV_NAME/bin/activate"

bootstrap/dev/venv.sh

@@ -0,0 +1,13 @@
+#!/bin/sh -xe
+# Developer virtualenv setup for Let's Encrypt client
+
+export VENV_ARGS="--python python2"
+
+./bootstrap/dev/_venv_common.sh \
+  -r requirements.txt \
+  -e acme[testing] \
+  -e .[dev,docs,testing] \
+  -e letsencrypt-apache \
+  -e letsencrypt-nginx \
+  -e letshelp-letsencrypt \
+  -e letsencrypt-compatibility-test

bootstrap/dev/venv3.sh

@@ -0,0 +1,8 @@
+#!/bin/sh -xe
+# Developer Python3 virtualenv setup for Let's Encrypt
+
+export VENV_NAME="${VENV_NAME:-venv3}"
+export VENV_ARGS="--python python3"
+
+./bootstrap/dev/_venv_common.sh \
+  -e acme[testing] \

bootstrap/freebsd.sh

@@ -0,0 +1,8 @@
+#!/bin/sh -xe
+
+pkg install -Ay \
+  git \
+  python \
+  py27-virtualenv \
+  augeas \
+  libffi \

bootstrap/mac.sh

@@ -1,2 +1,8 @@
 #!/bin/sh
+if ! hash brew 2>/dev/null; then
+    echo "Homebrew Not Installed\nDownloading..."
+    ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+fi
+
 brew install augeas
+brew install dialog

docs/api/display.rst

@@ -21,9 +21,3 @@
 
 .. automodule:: letsencrypt.display.enhancements
    :members:
-
-:mod:`letsencrypt.display.revocation`
-=====================================
-
-.. automodule:: letsencrypt.display.revocation
-   :members:

docs/api/recovery_token.rst

@@ -1,5 +0,0 @@
-:mod:`letsencrypt.recovery_token`
---------------------------------------------------
-
-.. automodule:: letsencrypt.recovery_token
-   :members:

docs/api/revoker.rst

@@ -1,5 +0,0 @@
-:mod:`letsencrypt.revoker`
---------------------------
-
-.. automodule:: letsencrypt.revoker
-   :members:

docs/conf.py

@@ -30,7 +30,7 @@ here = os.path.abspath(os.path.dirname(__file__))
 # read version number (and other metadata) from package init
 init_fn = os.path.join(here, '..', 'letsencrypt', '__init__.py')
 with codecs.open(init_fn, encoding='utf8') as fd:
-    meta = dict(re.findall(r"""__([a-z]+)__ = "([^"]+)""", fd.read()))
+    meta = dict(re.findall(r"""__([a-z]+)__ = '([^']+)""", fd.read()))
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -54,6 +54,7 @@ extensions = [
     'sphinx.ext.coverage',
     'sphinx.ext.viewcode',
     'repoze.sphinx.autointerface',
+    'sphinxcontrib.programoutput',
 ]
 
 autodoc_member_order = 'bysource'
@@ -283,7 +284,12 @@ latex_documents = [
 # (source start file, name, description, authors, manual section).
 man_pages = [
     ('index', 'letsencrypt', u'Let\'s Encrypt Documentation',
-     [u'Let\'s Encrypt Project'], 1)
+     [project], 7),
+    ('man/letsencrypt', 'letsencrypt', u'letsencrypt script documentation',
+     [project], 1),
+    ('man/letsencrypt-renewer', 'letsencrypt-renewer',
+     u'letsencrypt-renewer script documentation', [project], 1),
+    ('man/jws', 'jws', u'jws script documentation', [project], 1),
 ]
 
 # If true, show URL addresses after external links.

docs/contributing.rst

@@ -7,38 +7,37 @@ Contributing
 Hacking
 =======
 
-Start by :doc:`installing dependencies and setting up Let's Encrypt
-<using>`.
+All changes in your pull request **must** have 100% unit test coverage, pass
+our `integration`_ tests, **and** be compliant with the
+:ref:`coding style <coding-style>`.
 
-When you're done activate the virtualenv:
+
+Bootstrap
+---------
+
+Start by :ref:`installing Let's Encrypt prerequisites
+<prerequisites>`. Then run:
 
 .. code-block:: shell
 
-   source ./venv/bin/activate
+   ./bootstrap/dev/venv.sh
 
-This step should prepend you prompt with ``(venv)`` and save you from
-typing ``./venv/bin/...``. It is also required to run some of the
-`testing`_ tools. Virtualenv can be disabled at any time by typing
-``deactivate``. More information can be found in `virtualenv
+Activate the virtualenv:
+
+.. code-block:: shell
+
+   source ./$VENV_NAME/bin/activate
+
+This step should prepend you prompt with ``($VENV_NAME)`` and save you
+from typing ``./$VENV_NAME/bin/...``. It is also required to run some
+of the `testing`_ tools. Virtualenv can be disabled at any time by
+typing ``deactivate``. More information can be found in `virtualenv
 documentation`_.
 
-Install the development packages:
-
-.. code-block:: shell
-
-   pip install -r requirements.txt -e acme -e .[dev,docs,testing] -e letsencrypt-apache -e letsencrypt-nginx
-
-.. note:: `-e` (short for `--editable`) turns on *editable mode* in
-          which any source code changes in the current working
-          directory are "live" and no further `pip install ...`
-          invocations are necessary while developing.
-
-          This is roughly equivalent to `python setup.py develop`. For
-          more info see `man pip`.
-
-The code base, including your pull requests, **must** have 100% unit
-test coverage, pass our `integration`_ tests **and** be compliant with
-the :ref:`coding style <coding-style>`.
+Note that packages are installed in so called *editable mode*, in
+which any source code changes in the current working directory are
+"live" and no further ``./bootstrap/dev/venv.sh`` or ``pip install
+...`` invocations are necessary while developing.
 
 .. _`virtualenv documentation`: https://virtualenv.pypa.io
 
@@ -52,7 +51,8 @@ The following tools are there to help you:
   before submitting a new pull request.
 
 - ``tox -e cover`` checks the test coverage only. Calling the
-  ``./tox-cover.sh`` script directly might be a bit quicker, though.
+  ``./tox.cover.sh`` script directly (or even ``./tox.cover.sh $pkg1
+  $pkg2 ...`` for any subpackages) might be a bit quicker, though.
 
 - ``tox -e lint`` checks the style of the whole project, while
   ``pylint --rcfile=.pylintrc path`` will check a single file or
@@ -60,29 +60,33 @@ The following tools are there to help you:
 
 - For debugging, we recommend ``pip install ipdb`` and putting
   ``import ipdb; ipdb.set_trace()`` statement inside the source
-  code. Alternatively, you can use Python'd standard library `pdb`,
+  code. Alternatively, you can use Python's standard library `pdb`,
   but you won't get TAB completion...
 
 
 Integration
 ~~~~~~~~~~~
+Mac OS X users: Run `./tests/mac-bootstrap.sh` instead of `boulder-start.sh` to
+install dependencies, configure the environment, and start boulder.
 
-First, install `Go`_ 1.4 and start Boulder_, an ACME CA server::
+Otherwise, install `Go`_ 1.5, libtool-ltdl, mariadb-server and
+rabbitmq-server and then start Boulder_, an ACME CA server::
 
   ./tests/boulder-start.sh
 
 The script will download, compile and run the executable; please be
 patient - it will take some time... Once its ready, you will see
-``Server running, listening on 127.0.0.1:4000...``. You may now run
-(in a separate terminal)::
+``Server running, listening on 127.0.0.1:4000...``. Add an
+``/etc/hosts`` entry pointing ``le.wtf`` to 127.0.0.1.  You may now
+run (in a separate terminal)::
 
   ./tests/boulder-integration.sh && echo OK || echo FAIL
 
-If you would like to test `lesencrypt_nginx` plugin (highly
+If you would like to test `letsencrypt_nginx` plugin (highly
 encouraged) make sure to install prerequisites as listed in
-``tests/integration/nginx.sh``:
+``letsencrypt-nginx/tests/boulder-integration.sh``:
 
-.. include:: ../tests/integration/nginx.sh
+.. include:: ../letsencrypt-nginx/tests/boulder-integration.sh
    :start-line: 1
    :end-line: 2
    :code: shell
@@ -121,6 +125,27 @@ Support for other Linux distributions coming soon.
 .. _related issue: https://github.com/ClusterHQ/flocker/issues/516
 
 
+Docker
+------
+
+OSX users will probably find it easiest to set up a Docker container for
+development. Let's Encrypt comes with a Dockerfile (``Dockerfile-dev``)
+for doing so. To use Docker on OSX, install and setup docker-machine using the
+instructions at https://docs.docker.com/installation/mac/.
+
+To build the development Docker image::
+
+  docker build -t letsencrypt -f Dockerfile-dev .
+
+Now run tests inside the Docker image:
+
+.. code-block:: shell
+
+  docker run -it letsencrypt bash
+  cd src
+  tox -e py27
+
+
 Code components and layout
 ==========================
 
@@ -242,6 +267,22 @@ Please:
 .. _PEP 8 - Style Guide for Python Code:
   https://www.python.org/dev/peps/pep-0008
 
+Submitting a pull request
+=========================
+
+Steps:
+
+1. Write your code!
+2. Make sure your environment is set up properly and that you're in your
+   virtualenv. You can do this by running ``./bootstrap/dev/venv.sh``.
+   (this is a **very important** step)
+3. Run ``./pep8.travis.sh`` to do a cursory check of your code style.
+   Fix any errors.
+4. Run ``tox -e lint`` to check for pylint errors. Fix any errors.
+5. Run ``tox`` to run the entire test suite including coverage. Fix any errors.
+6. If your code touches communication with an ACME server/Boulder, you
+   should run the integration tests, see `integration`_.
+7. Submit the PR.
 
 Updating the documentation
 ==========================

docs/man/jws.rst

@@ -0,0 +1 @@
+.. program-output:: jws --help all

docs/man/letsencrypt-renewer.rst

@@ -0,0 +1 @@
+.. program-output:: letsencrypt-renewer --help

docs/man/letsencrypt.rst

@@ -0,0 +1 @@
+.. program-output:: letsencrypt --help all

docs/pkgs/letsencrypt_compatibility_test.rst

@@ -0,0 +1,53 @@
+:mod:`letsencrypt_compatibility_test`
+-------------------------------------
+
+.. automodule:: letsencrypt_compatibility_test
+   :members:
+
+:mod:`letsencrypt_compatibility_test.errors`
+============================================
+
+.. automodule:: letsencrypt_compatibility_test.errors
+   :members:
+
+:mod:`letsencrypt_compatibility_test.interfaces`
+================================================
+
+.. automodule:: letsencrypt_compatibility_test.interfaces
+   :members:
+
+:mod:`letsencrypt_compatibility_test.test_driver`
+=================================================
+
+.. automodule:: letsencrypt_compatibility_test.test_driver
+   :members:
+
+:mod:`letsencrypt_compatibility_test.util`
+==========================================
+
+.. automodule:: letsencrypt_compatibility_test.util
+   :members:
+
+:mod:`letsencrypt_compatibility_test.configurators`
+===================================================
+
+.. automodule:: letsencrypt_compatibility_test.configurators
+   :members:
+
+:mod:`letsencrypt_compatibility_test.configurators.apache`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. automodule:: letsencrypt_compatibility_test.configurators.apache
+   :members:
+
+:mod:`letsencrypt_compatibility_test.configurators.apache.apache24`
+-------------------------------------------------------------------
+
+.. automodule:: letsencrypt_compatibility_test.configurators.apache.apache24
+   :members:
+
+:mod:`letsencrypt_compatibility_test.configurators.apache.common`
+-------------------------------------------------------------------
+
+.. automodule:: letsencrypt_compatibility_test.configurators.apache.common
+   :members:

docs/pkgs/letshelp_letsencrypt.rst

@@ -0,0 +1,11 @@
+:mod:`letshelp_letsencrypt`
+---------------------------
+
+.. automodule:: letshelp_letsencrypt
+   :members:
+
+:mod:`letshelp_letsencrypt.apache`
+==================================
+
+.. automodule:: letshelp_letsencrypt.apache
+   :members:

docs/using.rst

@@ -42,6 +42,8 @@ above method instead.
    https://github.com/letsencrypt/letsencrypt/archive/master.zip
 
 
+.. _prerequisites:
+
 Prerequisites
 =============
 
@@ -83,7 +85,7 @@ Mac OSX
 
 .. code-block:: shell
 
-   sudo ./bootstrap/mac.sh
+   ./bootstrap/mac.sh
 
 
 Fedora
@@ -102,15 +104,32 @@ Centos 7
    sudo ./bootstrap/centos.sh
 
 
+FreeBSD
+-------
+
+.. code-block:: shell
+
+   sudo ./bootstrap/freebsd.sh
+
+Bootstrap script for FreeBSD uses ``pkg`` for package installation,
+i.e. it does not use ports.
+
+FreeBSD by default uses ``tcsh``. In order to activate virtulenv (see
+below), you will need a compatbile shell, e.g. ``pkg install bash &&
+bash``.
+
+
 Installation
 ============
 
 .. "pip install acme" doesn't search for "acme" in cwd, just like "pip
-   install -e acme" does
+   install -e acme" does; `-U setuptools pip` necessary for #722
 
 .. code-block:: shell
 
    virtualenv --no-site-packages -p python2 venv
+   ./venv/bin/pip install -U setuptools
+   ./venv/bin/pip install -U pip
    ./venv/bin/pip install -r requirements.txt acme/ . letsencrypt-apache/ letsencrypt-nginx/
 
 .. warning:: Please do **not** use ``python setup.py install``. Please
@@ -129,7 +148,7 @@ To get a new certificate run:
 
 .. code-block:: shell
 
-   ./venv/bin/letsencrypt auth
+   sudo ./venv/bin/letsencrypt auth
 
 The ``letsencrypt`` commandline tool has a builtin help:
 

examples/dev-cli.ini

@@ -9,6 +9,7 @@ domains = example.com
 
 text = True
 agree-eula = True
+agree-tos = True
 debug = True
 # Unfortunately, it's not possible to specify "verbose" multiple times
 # (correspondingly to -vvvvvv)

examples/generate-csr.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # This script generates a simple SAN CSR to be used with Let's Encrypt
-# CA. Mostly intedened for "auth --csr" testing, but, since its easily
-# auditable, feel free to adjust it and use on you production web
+# CA. Mostly intended for "auth --csr" testing, but, since it's easily
+# auditable, feel free to adjust it and use it on your production web
 # server.
 
 if [ "$#" -lt 1 ]

letsencrypt-apache/LICENSE.txt

@@ -0,0 +1,190 @@
+   Copyright 2015 Electronic Frontier Foundation and others
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

letsencrypt-apache/MANIFEST.in

@@ -1,2 +1,4 @@
+include LICENSE.txt
+include README.rst
 recursive-include letsencrypt_apache/tests/testdata *
 include letsencrypt_apache/options-ssl-apache.conf

letsencrypt-apache/README.rst

@@ -0,0 +1 @@
+Apache plugin for Let's Encrypt client

letsencrypt-apache/letsencrypt_apache/augeas_configurator.py

@@ -3,6 +3,7 @@ import logging
 
 import augeas
 
+from letsencrypt import errors
 from letsencrypt import reverter
 from letsencrypt.plugins import common
 
@@ -23,7 +24,6 @@ class AugeasConfigurator(common.Plugin):
     :type reverter: :class:`letsencrypt.reverter.Reverter`
 
     """
-
     def __init__(self, *args, **kwargs):
         super(AugeasConfigurator, self).__init__(*args, **kwargs)
 
@@ -38,13 +38,16 @@ class AugeasConfigurator(common.Plugin):
         # because this will change the underlying configuration and potential
         # vhosts
         self.reverter = reverter.Reverter(self.config)
-        self.reverter.recovery_routine()
+        self.recovery_routine()
 
     def check_parsing_errors(self, lens):
         """Verify Augeas can parse all of the lens files.
 
         :param str lens: lens to check for errors
 
+        :raises .errors.PluginError: If there has been an error in parsing with
+            the specified lens.
+
         """
         error_files = self.aug.match("/augeas//error")
 
@@ -54,11 +57,13 @@ class AugeasConfigurator(common.Plugin):
             lens_path = self.aug.get(path + "/lens")
             # As aug.get may return null
             if lens_path and lens in lens_path:
-                logger.error(
+                msg = (
                     "There has been an error in parsing the file (%s): %s",
                     # Strip off /augeas/files and /error
                     path[13:len(path) - 6], self.aug.get(path + "/message"))
+                raise errors.PluginError(msg)
 
+    # TODO: Cleanup this function
     def save(self, title=None, temporary=False):
         """Saves all changes to the configuration files.
 
@@ -73,6 +78,9 @@ class AugeasConfigurator(common.Plugin):
         :param bool temporary: Indicates whether the changes made will
             be quickly reversed in the future (ie. challenges)
 
+        :raises .errors.PluginError: If there was an error in Augeas, in an
+            attempt to save the configuration, or an error creating a checkpoint
+
         """
         save_state = self.aug.get("/augeas/save")
         self.aug.set("/augeas/save", "noop")
@@ -85,7 +93,8 @@ class AugeasConfigurator(common.Plugin):
             self._log_save_errors(ex_errs)
             # Erase Save Notes
             self.save_notes = ""
-            return False
+            raise errors.PluginError(
+                "Error saving files, check logs for more info.")
 
         # Retrieve list of modified files
         # Note: Noop saves can cause the file to be listed twice, I used a
@@ -99,22 +108,26 @@ class AugeasConfigurator(common.Plugin):
             for path in save_paths:
                 save_files.add(self.aug.get(path)[6:])
 
-            # Create Checkpoint
-            if temporary:
-                self.reverter.add_to_temp_checkpoint(
-                    save_files, self.save_notes)
-            else:
-                self.reverter.add_to_checkpoint(save_files, self.save_notes)
+            try:
+                # Create Checkpoint
+                if temporary:
+                    self.reverter.add_to_temp_checkpoint(
+                        save_files, self.save_notes)
+                else:
+                    self.reverter.add_to_checkpoint(save_files, self.save_notes)
+            except errors.ReverterError as err:
+                raise errors.PluginError(str(err))
 
         if title and not temporary:
-            self.reverter.finalize_checkpoint(title)
+            try:
+                self.reverter.finalize_checkpoint(title)
+            except errors.ReverterError as err:
+                raise errors.PluginError(str(err))
 
         self.aug.set("/augeas/save", save_state)
         self.save_notes = ""
         self.aug.save()
 
-        return True
-
     def _log_save_errors(self, ex_errs):
         """Log errors due to bad Augeas save.
 
@@ -135,14 +148,26 @@ class AugeasConfigurator(common.Plugin):
 
         Reverts all modified files that have not been saved as a checkpoint
 
+        :raises .errors.PluginError: If unable to recover the configuration
+
         """
-        self.reverter.recovery_routine()
+        try:
+            self.reverter.recovery_routine()
+        except errors.ReverterError as err:
+            raise errors.PluginError(str(err))
         # Need to reload configuration after these changes take effect
         self.aug.load()
 
     def revert_challenge_config(self):
-        """Used to cleanup challenge configurations."""
-        self.reverter.revert_temporary_config()
+        """Used to cleanup challenge configurations.
+
+        :raises .errors.PluginError: If unable to revert the challenge config.
+
+        """
+        try:
+            self.reverter.revert_temporary_config()
+        except errors.ReverterError as err:
+            raise errors.PluginError(str(err))
         self.aug.load()
 
     def rollback_checkpoints(self, rollback=1):
@@ -150,10 +175,24 @@ class AugeasConfigurator(common.Plugin):
 
         :param int rollback: Number of checkpoints to revert
 
+        :raises .errors.PluginError: If there is a problem with the input or
+            the function is unable to correctly revert the configuration
+
         """
-        self.reverter.rollback_checkpoints(rollback)
+        try:
+            self.reverter.rollback_checkpoints(rollback)
+        except errors.ReverterError as err:
+            raise errors.PluginError(str(err))
         self.aug.load()
 
     def view_config_changes(self):
-        """Show all of the configuration changes that have taken place."""
-        self.reverter.view_config_changes()
+        """Show all of the configuration changes that have taken place.
+
+        :raises .errors.PluginError: If there is a problem while processing
+            the checkpoints directories.
+
+        """
+        try:
+            self.reverter.view_config_changes()
+        except errors.ReverterError as err:
+            raise errors.PluginError(str(err))

letsencrypt-apache/letsencrypt_apache/constants.py

@@ -6,6 +6,7 @@ CLI_DEFAULTS = dict(
     server_root="/etc/apache2",
     ctl="apache2ctl",
     enmod="a2enmod",
+    dismod="a2dismod",
     init_script="/etc/init.d/apache2",
     le_vhost_ext="-le-ssl.conf",
 )
@@ -20,5 +21,5 @@ MOD_SSL_CONF_SRC = pkg_resources.resource_filename(
 distribution."""
 
 REWRITE_HTTPS_ARGS = [
-    "^.*$", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,R=permanent]"]
+    "^", "https://%{SERVER_NAME}%{REQUEST_URI}", "[L,QSA,R=permanent]"]
 """Apache rewrite rule arguments used for redirections to https vhost"""

letsencrypt-apache/letsencrypt_apache/display_ops.py

@@ -60,9 +60,9 @@ def _vhost_menu(domain, vhosts):
 
     choices = []
     for vhost in vhosts:
-        if len(vhost.names) == 1:
-            disp_name = next(iter(vhost.names))
-        elif len(vhost.names) == 0:
+        if len(vhost.get_names()) == 1:
+            disp_name = next(iter(vhost.get_names()))
+        elif len(vhost.get_names()) == 0:
             disp_name = ""
         else:
             disp_name = "Multiple Names"

letsencrypt-apache/letsencrypt_apache/dvsni.py

@@ -3,6 +3,7 @@ import os
 
 from letsencrypt.plugins import common
 
+from letsencrypt_apache import obj
 from letsencrypt_apache import parser
 
 
@@ -44,28 +45,24 @@ class ApacheDvsni(common.Dvsni):
 
 """
 
+    def __init__(self, *args, **kwargs):
+        super(ApacheDvsni, self).__init__(*args, **kwargs)
+
+        self.challenge_conf = os.path.join(
+            self.configurator.conf("server-root"),
+            "le_dvsni_cert_challenge.conf")
+
     def perform(self):
-        """Peform a DVSNI challenge."""
+        """Perform a DVSNI challenge."""
         if not self.achalls:
             return []
         # Save any changes to the configuration as a precaution
         # About to make temporary changes to the config
         self.configurator.save()
 
-        addresses = []
-        default_addr = "*:443"
-        for achall in self.achalls:
-            vhost = self.configurator.choose_vhost(achall.domain)
-
-            # TODO - @jdkasten review this code to make sure it makes sense
-            self.configurator.make_server_sni_ready(vhost, default_addr)
-
-            for addr in vhost.addrs:
-                if "_default_" == addr.get_addr():
-                    addresses.append([default_addr])
-                    break
-            else:
-                addresses.append(list(vhost.addrs))
+        # Prepare the server for HTTPS
+        self.configurator.prepare_server_https(
+            str(self.configurator.config.dvsni_port), True)
 
         responses = []
 
@@ -74,25 +71,32 @@ class ApacheDvsni(common.Dvsni):
             responses.append(self._setup_challenge_cert(achall))
 
         # Setup the configuration
-        self._mod_config(addresses)
+        dvsni_addrs = self._mod_config()
+        self.configurator.make_addrs_sni_ready(dvsni_addrs)
 
         # Save reversible changes
         self.configurator.save("SNI Challenge", True)
 
         return responses
 
-    def _mod_config(self, ll_addrs):
+    def _mod_config(self):
         """Modifies Apache config files to include challenge vhosts.
 
         Result: Apache config includes virtual servers for issued challs
 
-        :param list ll_addrs: list of list of `~.common.Addr` to apply
+        :returns: All DVSNI addresses used
+        :rtype: set
 
         """
-        # TODO: Use ip address of existing vhost instead of relying on FQDN
+        dvsni_addrs = set()
         config_text = "<IfModule mod_ssl.c>\n"
-        for idx, lis in enumerate(ll_addrs):
-            config_text += self._get_config_text(self.achalls[idx], lis)
+
+        for achall in self.achalls:
+            achall_addrs = self.get_dvsni_addrs(achall)
+            dvsni_addrs.update(achall_addrs)
+
+            config_text += self._get_config_text(achall, achall_addrs)
+
         config_text += "</IfModule>\n"
 
         self._conf_include_check(self.configurator.parser.loc["default"])
@@ -102,6 +106,25 @@ class ApacheDvsni(common.Dvsni):
         with open(self.challenge_conf, "w") as new_conf:
             new_conf.write(config_text)
 
+        return dvsni_addrs
+
+    def get_dvsni_addrs(self, achall):
+        """Return the Apache addresses needed for DVSNI."""
+        vhost = self.configurator.choose_vhost(achall.domain)
+
+        # TODO: Checkout _default_ rules.
+        dvsni_addrs = set()
+        default_addr = obj.Addr(("*", str(self.configurator.config.dvsni_port)))
+
+        for addr in vhost.addrs:
+            if "_default_" == addr.get_addr():
+                dvsni_addrs.add(default_addr)
+            else:
+                dvsni_addrs.add(
+                    addr.get_sni_addr(self.configurator.config.dvsni_port))
+
+        return dvsni_addrs
+
     def _conf_include_check(self, main_config):
         """Adds DVSNI challenge conf file into configuration.
 
@@ -125,7 +148,7 @@ class ApacheDvsni(common.Dvsni):
         :type achall: :class:`letsencrypt.achallenges.DVSNI`
 
         :param list ip_addrs: addresses of challenged domain
-            :class:`list` of type `~.common.Addr`
+            :class:`list` of type `~.obj.Addr`
 
         :returns: virtual host configuration text
         :rtype: str
@@ -140,8 +163,9 @@ class ApacheDvsni(common.Dvsni):
         # parses it as "\n"... c.f.:
         # https://docs.python.org/2.7/reference/lexical_analysis.html
         return self.VHOST_TEMPLATE.format(
-            vhost=ips, server_name=achall.nonce_domain,
-            ssl_options_conf_path=self.configurator.parser.loc["ssl_options"],
+            vhost=ips,
+            server_name=achall.gen_response(achall.account_key).z_domain,
+            ssl_options_conf_path=self.configurator.mod_ssl_conf,
             cert_path=self.get_cert_path(achall),
             key_path=self.get_key_path(achall),
             document_root=document_root).replace("\n", os.linesep)

letsencrypt-apache/letsencrypt_apache/obj.py

@@ -1,4 +1,92 @@
 """Module contains classes used by the Apache Configurator."""
+import re
+
+from letsencrypt.plugins import common
+
+
+class Addr(common.Addr):
+    """Represents an Apache address."""
+    def __eq__(self, other):
+        """This is defined as equalivalent within Apache.
+
+        ip_addr:* == ip_addr
+
+        """
+        if isinstance(other, self.__class__):
+            return ((self.tup == other.tup) or
+                    (self.tup[0] == other.tup[0] and
+                     self.is_wildcard() and other.is_wildcard()))
+        return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def _addr_less_specific(self, addr):
+        """Returns if addr.get_addr() is more specific than self.get_addr()."""
+        # pylint: disable=protected-access
+        return addr._rank_specific_addr() > self._rank_specific_addr()
+
+    def _rank_specific_addr(self):
+        """Returns numerical rank for get_addr()
+
+        :returns: 2 - FQ, 1 - wildcard, 0 - _default_
+        :rtype: int
+
+        """
+        if self.get_addr() == "_default_":
+            return 0
+        elif self.get_addr() == "*":
+            return 1
+        else:
+            return 2
+
+    def conflicts(self, addr):
+        r"""Returns if address could conflict with correct function of self.
+
+        Could addr take away service provided by self within Apache?
+
+        .. note::IP Address is more important than wildcard.
+            Connection from 127.0.0.1:80 with choices of *:80 and 127.0.0.1:*
+            chooses 127.0.0.1:\*
+
+        .. todo:: Handle domain name addrs...
+
+        Examples:
+
+        =========================================  =====
+        ``127.0.0.1:\*.conflicts(127.0.0.1:443)``  True
+        ``127.0.0.1:443.conflicts(127.0.0.1:\*)``  False
+        ``\*:443.conflicts(\*:80)``                False
+        ``_default_:443.conflicts(\*:443)``        True
+        =========================================  =====
+
+        """
+        if self._addr_less_specific(addr):
+            return True
+        elif self.get_addr() == addr.get_addr():
+            if self.is_wildcard() or self.get_port() == addr.get_port():
+                return True
+        return False
+
+    def is_wildcard(self):
+        """Returns if address has a wildcard port."""
+        return self.tup[1] == "*" or not self.tup[1]
+
+    def get_sni_addr(self, port):
+        """Returns the least specific address that resolves on the port.
+
+        Examples:
+
+        - ``1.2.3.4:443`` -> ``1.2.3.4:<port>``
+        - ``1.2.3.4:*`` -> ``1.2.3.4:*``
+
+        :param str port: Desired port
+
+        """
+        if self.is_wildcard():
+            return self
+
+        return self.get_addr_obj(port)
 
 
 class VirtualHost(object):  # pylint: disable=too-few-public-methods
@@ -8,39 +96,57 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
     :ivar str path: Augeas path to virtual host
     :ivar set addrs: Virtual Host addresses (:class:`set` of
         :class:`common.Addr`)
-    :ivar set names: Server names/aliases of vhost
+    :ivar str name: ServerName of VHost
+    :ivar list aliases: Server aliases of vhost
         (:class:`list` of :class:`str`)
 
     :ivar bool ssl: SSLEngine on in vhost
     :ivar bool enabled: Virtual host is enabled
 
+    https://httpd.apache.org/docs/2.4/vhosts/details.html
+
+    .. todo:: Any vhost that includes the magic _default_ wildcard is given the
+              same ServerName as the main server.
+
     """
-    def __init__(self, filep, path, addrs, ssl, enabled, names=None):
+    # ?: is used for not returning enclosed characters
+    strip_name = re.compile(r"^(?:.+://)?([^ :$]*)")
+
+    def __init__(self, filep, path, addrs, ssl, enabled, name=None, aliases=None):
         # pylint: disable=too-many-arguments
         """Initialize a VH."""
         self.filep = filep
         self.path = path
         self.addrs = addrs
-        self.names = set() if names is None else set(names)
+        self.name = name
+        self.aliases = aliases if aliases is not None else set()
         self.ssl = ssl
         self.enabled = enabled
 
-    def add_name(self, name):
-        """Add name to vhost."""
-        self.names.add(name)
+    def get_names(self):
+        """Return a set of all names."""
+        all_names = set()
+        all_names.update(self.aliases)
+        # Strip out any scheme:// and <port> field from servername
+        if self.name is not None:
+            all_names.add(VirtualHost.strip_name.findall(self.name)[0])
+
+        return all_names
 
     def __str__(self):
         return (
             "File: {filename}\n"
             "Vhost path: {vhpath}\n"
             "Addresses: {addrs}\n"
-            "Names: {names}\n"
+            "Name: {name}\n"
+            "Aliases: {aliases}\n"
             "TLS Enabled: {tls}\n"
             "Site Enabled: {active}".format(
                 filename=self.filep,
                 vhpath=self.path,
                 addrs=", ".join(str(addr) for addr in self.addrs),
-                names=", ".join(name for name in self.names),
+                name=self.name if self.name is not None else "",
+                aliases=", ".join(name for name in self.aliases),
                 tls="Yes" if self.ssl else "No",
                 active="Yes" if self.enabled else "No"))
 
@@ -48,7 +154,73 @@ class VirtualHost(object):  # pylint: disable=too-few-public-methods
         if isinstance(other, self.__class__):
             return (self.filep == other.filep and self.path == other.path and
                     self.addrs == other.addrs and
-                    self.names == other.names and
+                    self.get_names() == other.get_names() and
                     self.ssl == other.ssl and self.enabled == other.enabled)
 
         return False
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def conflicts(self, addrs):
+        """See if vhost conflicts with any of the addrs.
+
+        This determines whether or not these addresses would/could overwrite
+        the vhost addresses.
+
+        :param addrs: Iterable Addresses
+        :type addrs: Iterable :class:~obj.Addr
+
+        :returns: If addresses conflicts with vhost
+        :rtype: bool
+
+        """
+        for pot_addr in addrs:
+            for addr in self.addrs:
+                if addr.conflicts(pot_addr):
+                    return True
+        return False
+
+    def same_server(self, vhost):
+        """Determines if the vhost is the same 'server'.
+
+        Used in redirection - indicates whether or not the two virtual hosts
+        serve on the exact same IP combinations, but different ports.
+
+        .. todo:: Handle _default_
+
+        """
+
+        if vhost.get_names() != self.get_names():
+            return False
+
+        # If equal and set is not empty... assume same server
+        if self.name is not None or self.aliases:
+            return True
+
+        # Both sets of names are empty.
+
+        # Make conservative educated guess... this is very restrictive
+        # Consider adding more safety checks.
+        if len(vhost.addrs) != len(self.addrs):
+            return False
+
+        # already_found acts to keep everything very conservative.
+        # Don't allow multiple ip:ports in same set.
+        already_found = set()
+
+        for addr in vhost.addrs:
+            for local_addr in self.addrs:
+                if (local_addr.get_addr() == addr.get_addr() and
+                        local_addr != addr and
+                        local_addr.get_addr() not in already_found):
+
+                    # This intends to make sure we aren't double counting...
+                    # e.g. 127.0.0.1:* - We require same number of addrs
+                    #  currently
+                    already_found.add(local_addr.get_addr())
+                    break
+            else:
+                return False
+
+        return True

letsencrypt-apache/letsencrypt_apache/parser.py

@@ -1,33 +1,177 @@
 """ApacheParser is a member object of the ApacheConfigurator class."""
+import fnmatch
+import itertools
+import logging
 import os
 import re
+import subprocess
 
 from letsencrypt import errors
 
 
+logger = logging.getLogger(__name__)
+
+
 class ApacheParser(object):
     """Class handles the fine details of parsing the Apache Configuration.
 
-    :ivar str root: Normalized abosulte path to the server root
+    .. todo:: Make parsing general... remove sites-available etc...
+
+    :ivar str root: Normalized absolute path to the server root
         directory. Without trailing slash.
+    :ivar str root: Server root
+    :ivar set modules: All module names that are currently enabled.
+    :ivar dict loc: Location to place directives, root - configuration origin,
+        default - user config file, name - NameVirtualHost,
 
     """
+    arg_var_interpreter = re.compile(r"\$\{[^ \}]*}")
+    fnmatch_chars = set(["*", "?", "\\", "[", "]"])
+
+    def __init__(self, aug, root, ctl):
+        # Note: Order is important here.
+
+        # This uses the binary, so it can be done first.
+        # https://httpd.apache.org/docs/2.4/mod/core.html#define
+        # https://httpd.apache.org/docs/2.4/mod/core.html#ifdefine
+        # This only handles invocation parameters and Define directives!
+        self.variables = {}
+        self.update_runtime_variables(ctl)
 
-    def __init__(self, aug, root, ssl_options):
-        # Find configuration root and make sure augeas can parse it.
         self.aug = aug
+        # Find configuration root and make sure augeas can parse it.
         self.root = os.path.abspath(root)
-        self.loc = self._set_locations(ssl_options)
+        self.loc = {"root": self._find_config_root()}
         self._parse_file(self.loc["root"])
 
+        # This problem has been fixed in Augeas 1.0
+        self.standardize_excl()
+
+        # Temporarily set modules to be empty, so that find_dirs can work
+        # https://httpd.apache.org/docs/2.4/mod/core.html#ifmodule
+        # This needs to come before locations are set.
+        self.modules = set()
+        self.init_modules()
+
+        # Set up rest of locations
+        self.loc.update(self._set_locations())
+
         # Must also attempt to parse sites-available or equivalent
         # Sites-available is not included naturally in configuration
         self._parse_file(os.path.join(self.root, "sites-available") + "/*")
 
-        # This problem has been fixed in Augeas 1.0
-        self.standardize_excl()
+    def init_modules(self):
+        """Iterates on the configuration until no new modules are loaded.
 
-    def add_dir_to_ifmodssl(self, aug_conf_path, directive, val):
+        ..todo:: This should be attempted to be done with a binary to avoid
+            the iteration issue.  Else... parse and enable mods at same time.
+
+        """
+        # Since modules are being initiated... clear existing set.
+        self.modules = set()
+        matches = self.find_dir("LoadModule")
+
+        iterator = iter(matches)
+        # Make sure prev_size != cur_size for do: while: iteration
+        prev_size = -1
+
+        while len(self.modules) != prev_size:
+            prev_size = len(self.modules)
+
+            for match_name, match_filename in itertools.izip(
+                    iterator, iterator):
+                self.modules.add(self.get_arg(match_name))
+                self.modules.add(
+                    os.path.basename(self.get_arg(match_filename))[:-2] + "c")
+
+    def update_runtime_variables(self, ctl):
+        """"
+
+        .. note:: Compile time variables (apache2ctl -V) are not used within the
+            dynamic configuration files.  These should not be parsed or
+            interpreted.
+
+        .. todo:: Create separate compile time variables... simply for arg_get()
+
+        """
+        stdout = self._get_runtime_cfg(ctl)
+
+        variables = dict()
+        matches = re.compile(r"Define: ([^ \n]*)").findall(stdout)
+        try:
+            matches.remove("DUMP_RUN_CFG")
+        except ValueError:
+            raise errors.PluginError("Unable to parse runtime variables")
+
+        for match in matches:
+            if match.count("=") > 1:
+                logger.error("Unexpected number of equal signs in "
+                             "apache2ctl -D DUMP_RUN_CFG")
+                raise errors.PluginError(
+                    "Error parsing Apache runtime variables")
+            parts = match.partition("=")
+            variables[parts[0]] = parts[2]
+
+        self.variables = variables
+
+    def _get_runtime_cfg(self, ctl):  # pylint: disable=no-self-use
+        """Get runtime configuration info.
+
+        :returns: stdout from DUMP_RUN_CFG
+
+        """
+        try:
+            proc = subprocess.Popen(
+                [ctl, "-D", "DUMP_RUN_CFG"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE)
+            stdout, stderr = proc.communicate()
+
+        except (OSError, ValueError):
+            logger.error(
+                "Error accessing %s for runtime parameters!%s", ctl, os.linesep)
+            raise errors.MisconfigurationError(
+                "Error accessing loaded Apache parameters: %s", ctl)
+        # Small errors that do not impede
+        if proc.returncode != 0:
+            logger.warn("Error in checking parameter list: %s", stderr)
+            raise errors.MisconfigurationError(
+                "Apache is unable to check whether or not the module is "
+                "loaded because Apache is misconfigured.")
+
+        return stdout
+
+    def filter_args_num(self, matches, args):  # pylint: disable=no-self-use
+        """Filter out directives with specific number of arguments.
+
+        This function makes the assumption that all related arguments are given
+        in order.  Thus /files/apache/directive[5]/arg[2] must come immediately
+        after /files/apache/directive[5]/arg[1]. Runs in 1 linear pass.
+
+        :param string matches: Matches of all directives with arg nodes
+        :param int args: Number of args you would like to filter
+
+        :returns: List of directives that contain # of arguments.
+            (arg is stripped off)
+
+        """
+        filtered = []
+        if args == 1:
+            for i in range(len(matches)):
+                if matches[i].endswith("/arg"):
+                    filtered.append(matches[i][:-4])
+        else:
+            for i in range(len(matches)):
+                if matches[i].endswith("/arg[%d]" % args):
+                    # Make sure we don't cause an IndexError (end of list)
+                    # Check to make sure arg + 1 doesn't exist
+                    if (i == (len(matches) - 1) or
+                            not matches[i + 1].endswith("/arg[%d]" % (args + 1))):
+                        filtered.append(matches[i][:-len("/arg[%d]" % args)])
+
+        return filtered
+
+    def add_dir_to_ifmodssl(self, aug_conf_path, directive, args):
         """Adds directive and value to IfMod ssl block.
 
         Adds given directive and value along configuration path within
@@ -35,8 +179,9 @@ class ApacheParser(object):
         the file, it is created.
 
         :param str aug_conf_path: Desired Augeas config path to add directive
-        :param str directive: Directive you would like to add
-        :param str val: Value of directive ie. Listen 443, 443 is the value
+        :param str directive: Directive you would like to add, e.g. Listen
+        :param args: Values of the directive; str "443" or list of str
+        :type args: list
 
         """
         # TODO: Add error checking code... does the path given even exist?
@@ -46,7 +191,11 @@ class ApacheParser(object):
         self.aug.insert(if_mod_path + "arg", "directive", False)
         nvh_path = if_mod_path + "directive[1]"
         self.aug.set(nvh_path, directive)
-        self.aug.set(nvh_path + "/arg", val)
+        if len(args) == 1:
+            self.aug.set(nvh_path + "/arg", args[0])
+        else:
+            for i, arg in enumerate(args):
+                self.aug.set("%s/arg[%d]" % (nvh_path, i + 1), arg)
 
     def _get_ifmod(self, aug_conf_path, mod):
         """Returns the path to <IfMod mod> and creates one if it doesn't exist.
@@ -65,7 +214,7 @@ class ApacheParser(object):
         # Strip off "arg" at end of first ifmod path
         return if_mods[0][:len(if_mods[0]) - 3]
 
-    def add_dir(self, aug_conf_path, directive, arg):
+    def add_dir(self, aug_conf_path, directive, args):
         """Appends directive to the end fo the file given by aug_conf_path.
 
         .. note:: Not added to AugeasConfigurator because it may depend
@@ -73,25 +222,29 @@ class ApacheParser(object):
 
         :param str aug_conf_path: Augeas configuration path to add directive
         :param str directive: Directive to add
-        :param str arg: Value of the directive. ie. Listen 443, 443 is arg
+        :param args: Value of the directive. ie. Listen 443, 443 is arg
+        :type args: list or str
 
         """
         self.aug.set(aug_conf_path + "/directive[last() + 1]", directive)
-        if isinstance(arg, list):
-            for i, value in enumerate(arg, 1):
+        if isinstance(args, list):
+            for i, value in enumerate(args, 1):
                 self.aug.set(
                     "%s/directive[last()]/arg[%d]" % (aug_conf_path, i), value)
         else:
-            self.aug.set(aug_conf_path + "/directive[last()]/arg", arg)
+            self.aug.set(aug_conf_path + "/directive[last()]/arg", args)
 
-    def find_dir(self, directive, arg=None, start=None):
+    def find_dir(self, directive, arg=None, start=None, exclude=True):
         """Finds directive in the configuration.
 
         Recursively searches through config files to find directives
         Directives should be in the form of a case insensitive regex currently
 
-        .. todo:: Add order to directives returned. Last directive comes last..
         .. todo:: arg should probably be a list
+        .. todo:: arg search currently only supports direct matching. It does
+            not handle the case of variables or quoted arguments. This should
+            be adapted to use a generic search for the directive and then do a
+            case-insensitive self.get_arg filter
 
         Note: Augeas is inherently case sensitive while Apache is case
         insensitive.  Augeas 1.0 allows case insensitive regexes like
@@ -101,20 +254,19 @@ class ApacheParser(object):
         compatibility.
 
         :param str directive: Directive to look for
-
         :param arg: Specific value directive must have, None if all should
                     be considered
         :type arg: str or None
 
         :param str start: Beginning Augeas path to begin looking
+        :param bool exclude: Whether or not to exclude directives based on
+            variables and enabled modules
 
         """
         # Cannot place member variable in the definition of the function so...
         if not start:
             start = get_aug_path(self.loc["root"])
 
-        # Debug code
-        # print "find_dir:", directive, "arg:", arg, " | Looking in:", start
         # No regexp code
         # if arg is None:
         #     matches = self.aug.match(start +
@@ -127,32 +279,109 @@ class ApacheParser(object):
         # includes = self.aug.match(start +
         # "//* [self::directive='Include']/* [label()='arg']")
 
+        regex = "(%s)|(%s)|(%s)" % (case_i(directive),
+                                    case_i("Include"),
+                                    case_i("IncludeOptional"))
+        matches = self.aug.match(
+            "%s//*[self::directive=~regexp('%s')]" % (start, regex))
+
+        if exclude:
+            matches = self._exclude_dirs(matches)
+
         if arg is None:
-            matches = self.aug.match(("%s//*[self::directive=~regexp('%s')]/arg"
-                                      % (start, directive)))
+            arg_suffix = "/arg"
         else:
-            matches = self.aug.match(("%s//*[self::directive=~regexp('%s')]/*"
-                                      "[self::arg=~regexp('%s')]" %
-                                      (start, directive, arg)))
+            arg_suffix = "/*[self::arg=~regexp('%s')]" % case_i(arg)
 
-        incl_regex = "(%s)|(%s)" % (case_i('Include'),
-                                    case_i('IncludeOptional'))
+        ordered_matches = []
 
-        includes = self.aug.match(("%s//* [self::directive=~regexp('%s')]/* "
-                                   "[label()='arg']" % (start, incl_regex)))
+        # TODO: Wildcards should be included in alphabetical order
+        # https://httpd.apache.org/docs/2.4/mod/core.html#include
+        for match in matches:
+            dir_ = self.aug.get(match).lower()
+            if dir_ == "include" or dir_ == "includeoptional":
+                # start[6:] to strip off /files
+                #print self._get_include_path(self.get_arg(match +"/arg")), directive, arg
+                ordered_matches.extend(self.find_dir(
+                    directive, arg,
+                    self._get_include_path(self.get_arg(match + "/arg")),
+                    exclude))
+            # This additionally allows Include
+            if dir_ == directive.lower():
+                ordered_matches.extend(self.aug.match(match + arg_suffix))
 
-        # for inc in includes:
-        #    print inc, self.aug.get(inc)
+        return ordered_matches
 
-        for include in includes:
-            # start[6:] to strip off /files
-            matches.extend(self.find_dir(
-                directive, arg, self._get_include_path(
-                    strip_dir(start[6:]), self.aug.get(include))))
+    def get_arg(self, match):
+        """Uses augeas.get to get argument value and interprets result.
 
-        return matches
+        This also converts all variables and parameters appropriately.
 
-    def _get_include_path(self, cur_dir, arg):
+        """
+        value = self.aug.get(match)
+
+        # No need to strip quotes for variables, as apache2ctl already does this
+        # but we do need to strip quotes for all normal arguments.
+
+        # Note: normal argument may be a quoted variable
+        # e.g. strip now, not later
+        value = value.strip("'\"")
+
+        variables = ApacheParser.arg_var_interpreter.findall(value)
+
+        for var in variables:
+            # Strip off ${ and }
+            try:
+                value = value.replace(var, self.variables[var[2:-1]])
+            except KeyError:
+                raise errors.PluginError("Error Parsing variable: %s" % var)
+
+        return value
+
+    def _exclude_dirs(self, matches):
+        """Exclude directives that are not loaded into the configuration."""
+        filters = [("ifmodule", self.modules), ("ifdefine", self.variables)]
+
+        valid_matches = []
+
+        for match in matches:
+            for filter_ in filters:
+                if not self._pass_filter(match, filter_):
+                    break
+            else:
+                valid_matches.append(match)
+        return valid_matches
+
+    def _pass_filter(self, match, filter_):
+        """Determine if directive passes a filter.
+
+        :param str match: Augeas path
+        :param list filter: list of tuples of form
+            [("lowercase if directive", set of relevant parameters)]
+
+        """
+        match_l = match.lower()
+        last_match_idx = match_l.find(filter_[0])
+
+        while last_match_idx != -1:
+            # Check args
+            end_of_if = match_l.find("/", last_match_idx)
+            # This should be aug.get (vars are not used e.g. parser.aug_get)
+            expression = self.aug.get(match[:end_of_if] + "/arg")
+
+            if expression.startswith("!"):
+                # Strip off "!"
+                if expression[1:] in filter_[1]:
+                    return False
+            else:
+                if expression not in filter_[1]:
+                    return False
+
+            last_match_idx = match_l.find(filter_[0], end_of_if)
+
+        return True
+
+    def _get_include_path(self, arg):
         """Converts an Apache Include directive into Augeas path.
 
         Converts an Apache Include directive argument into an Augeas
@@ -160,29 +389,12 @@ class ApacheParser(object):
 
         .. todo:: convert to use os.path.join()
 
-        :param str cur_dir: current working directory
-
         :param str arg: Argument of Include directive
 
         :returns: Augeas path string
         :rtype: str
 
         """
-        # Sanity check argument - maybe
-        # Question: what can the attacker do with control over this string
-        # Effect parse file... maybe exploit unknown errors in Augeas
-        # If the attacker can Include anything though... and this function
-        # only operates on Apache real config data... then the attacker has
-        # already won.
-        # Perhaps it is better to simply check the permissions on all
-        # included files?
-        # check_config to validate apache config doesn't work because it
-        # would create a race condition between the check and this input
-
-        # TODO: Maybe... although I am convinced we have lost if
-        # Apache files can't be trusted.  The augeas include path
-        # should be made to be exact.
-
         # Check to make sure only expected characters are used <- maybe remove
         # validChars = re.compile("[a-zA-Z0-9.*?_-/]*")
         # matchObj = validChars.match(arg)
@@ -190,60 +402,55 @@ class ApacheParser(object):
         #     logger.error("Error: Invalid regexp characters in %s", arg)
         #     return []
 
+        # Remove beginning and ending quotes
+        arg = arg.strip("'\"")
+
         # Standardize the include argument based on server root
         if not arg.startswith("/"):
-            arg = cur_dir + arg
-        # conf/ is a special variable for ServerRoot in Apache
-        elif arg.startswith("conf/"):
-            arg = self.root + arg[4:]
-        # TODO: Test if Apache allows ../ or ~/ for Includes
+            # Normpath will condense ../
+            arg = os.path.normpath(os.path.join(self.root, arg))
+        else:
+            arg = os.path.normpath(arg)
 
         # Attempts to add a transform to the file if one does not already exist
-        self._parse_file(arg)
+        if os.path.isdir(arg):
+            self._parse_file(os.path.join(arg, "*"))
+        else:
+            self._parse_file(arg)
 
         # Argument represents an fnmatch regular expression, convert it
         # Split up the path and convert each into an Augeas accepted regex
         # then reassemble
-        if "*" in arg or "?" in arg:
-            split_arg = arg.split("/")
-            for idx, split in enumerate(split_arg):
-                # * and ? are the two special fnmatch characters
-                if "*" in split or "?" in split:
-                    # Turn it into a augeas regex
-                    # TODO: Can this instead be an augeas glob instead of regex
-                    split_arg[idx] = ("* [label()=~regexp('%s')]" %
-                                      self.fnmatch_to_re(split))
-            # Reassemble the argument
-            arg = "/".join(split_arg)
+        split_arg = arg.split("/")
+        for idx, split in enumerate(split_arg):
+            if any(char in ApacheParser.fnmatch_chars for char in split):
+                # Turn it into a augeas regex
+                # TODO: Can this instead be an augeas glob instead of regex
+                split_arg[idx] = ("* [label()=~regexp('%s')]" %
+                                  self.fnmatch_to_re(split))
+        # Reassemble the argument
+        # Note: This also normalizes the argument /serverroot/ -> /serverroot
+        arg = "/".join(split_arg)
 
-        # If the include is a directory, just return the directory as a file
-        if arg.endswith("/"):
-            return get_aug_path(arg[:len(arg)-1])
         return get_aug_path(arg)
 
     def fnmatch_to_re(self, clean_fn_match):  # pylint: disable=no-self-use
         """Method converts Apache's basic fnmatch to regular expression.
 
+        Assumption - Configs are assumed to be well-formed and only writable by
+        privileged users.
+
+        https://apr.apache.org/docs/apr/2.0/apr__fnmatch_8h_source.html
+        http://apache2.sourcearchive.com/documentation/2.2.16-6/apr__fnmatch_8h_source.html
+
         :param str clean_fn_match: Apache style filename match, similar to globs
 
         :returns: regex suitable for augeas
         :rtype: str
 
         """
-        # Checkout fnmatch.py in venv/local/lib/python2.7/fnmatch.py
-        regex = ""
-        for letter in clean_fn_match:
-            if letter == '.':
-                regex = regex + r"\."
-            elif letter == '*':
-                regex = regex + ".*"
-            # According to apache.org ? shouldn't appear
-            # but in case it is valid...
-            elif letter == '?':
-                regex = regex + "."
-            else:
-                regex = regex + letter
-        return regex
+        # This strips off final /Z(?ms)
+        return fnmatch.translate(clean_fn_match)[:-7]
 
     def _parse_file(self, filepath):
         """Parse file with Augeas
@@ -318,15 +525,14 @@ class ApacheParser(object):
 
         self.aug.load()
 
-    def _set_locations(self, ssl_options):
+    def _set_locations(self):
         """Set default location for directives.
 
         Locations are given as file_paths
         .. todo:: Make sure that files are included
 
         """
-        root = self._find_config_root()
-        default = self._set_user_config_file(root)
+        default = self._set_user_config_file()
 
         temp = os.path.join(self.root, "ports.conf")
         if os.path.isfile(temp):
@@ -336,8 +542,7 @@ class ApacheParser(object):
             listen = default
             name = default
 
-        return {"root": root, "default": default, "listen": listen,
-                "name": name, "ssl_options": ssl_options}
+        return {"default": default, "listen": listen, "name": name}
 
     def _find_config_root(self):
         """Find the Apache Configuration Root file."""
@@ -349,7 +554,7 @@ class ApacheParser(object):
 
         raise errors.NoInstallationError("Could not find configuration root")
 
-    def _set_user_config_file(self, root):
+    def _set_user_config_file(self):
         """Set the appropriate user configuration file
 
         .. todo:: This will have to be updated for other distros versions
@@ -360,12 +565,11 @@ class ApacheParser(object):
         # Basic check to see if httpd.conf exists and
         # in hierarchy via direct include
         # httpd.conf was very common as a user file in Apache 2.2
-        if (os.path.isfile(os.path.join(self.root, 'httpd.conf')) and
-                self.find_dir(
-                    case_i("Include"), case_i("httpd.conf"), root)):
-            return os.path.join(self.root, 'httpd.conf')
+        if (os.path.isfile(os.path.join(self.root, "httpd.conf")) and
+                self.find_dir("Include", "httpd.conf", self.loc["root"])):
+            return os.path.join(self.root, "httpd.conf")
         else:
-            return os.path.join(self.root, 'apache2.conf')
+            return os.path.join(self.root, "apache2.conf")
 
 
 def case_i(string):
@@ -380,7 +584,7 @@ def case_i(string):
     :param str string: string to make case i regex
 
     """
-    return "".join(["["+c.upper()+c.lower()+"]"
+    return "".join(["[" + c.upper() + c.lower() + "]"
                     if c.isalpha() else c for c in re.escape(string)])
 
 
@@ -391,22 +595,3 @@ def get_aug_path(file_path):
 
     """
     return "/files%s" % file_path
-
-
-def strip_dir(path):
-    """Returns directory of file path.
-
-    .. todo:: Replace this with Python standard function
-
-    :param str path: path is a file path. not an augeas section or
-        directive path
-
-    :returns: directory
-    :rtype: str
-
-    """
-    index = path.rfind("/")
-    if index > 0:
-        return path[:index+1]
-    # No directory
-    return ""

letsencrypt-apache/letsencrypt_apache/tests/augeas_configurator_test.py

@@ -0,0 +1,115 @@
+"""Test for letsencrypt_apache.augeas_configurator."""
+import os
+import shutil
+import unittest
+
+import mock
+
+from letsencrypt import errors
+
+from letsencrypt_apache.tests import util
+
+
+class AugeasConfiguratorTest(util.ApacheTest):
+    """Test for Augeas Configurator base class."""
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(AugeasConfiguratorTest, self).setUp()
+
+        self.config = util.get_apache_configurator(
+            self.config_path, self.config_dir, self.work_dir)
+
+        self.vh_truth = util.get_vh_truth(
+            self.temp_dir, "debian_apache_2_4/two_vhost_80")
+
+    def tearDown(self):
+        shutil.rmtree(self.config_dir)
+        shutil.rmtree(self.work_dir)
+        shutil.rmtree(self.temp_dir)
+
+    def test_bad_parse(self):
+        # pylint: disable=protected-access
+        self.config.parser._parse_file(os.path.join(
+            self.config.parser.root, "conf-available", "bad_conf_file.conf"))
+        self.assertRaises(
+            errors.PluginError, self.config.check_parsing_errors, "httpd.aug")
+
+    def test_bad_save(self):
+        mock_save = mock.Mock()
+        mock_save.side_effect = IOError
+        self.config.aug.save = mock_save
+
+        self.assertRaises(errors.PluginError, self.config.save)
+
+    def test_bad_save_checkpoint(self):
+        self.config.reverter.add_to_checkpoint = mock.Mock(
+            side_effect=errors.ReverterError)
+        self.config.parser.add_dir(
+            self.vh_truth[0].path, "Test", "bad_save_ckpt")
+        self.assertRaises(errors.PluginError, self.config.save)
+
+    def test_bad_save_finalize_checkpoint(self):
+        self.config.reverter.finalize_checkpoint = mock.Mock(
+            side_effect=errors.ReverterError)
+        self.config.parser.add_dir(
+            self.vh_truth[0].path, "Test", "bad_save_ckpt")
+        self.assertRaises(errors.PluginError, self.config.save, "Title")
+
+    def test_finalize_save(self):
+        mock_finalize = mock.Mock()
+        self.config.reverter = mock_finalize
+        self.config.save("Example Title")
+
+        self.assertTrue(mock_finalize.is_called)
+
+    def test_recovery_routine(self):
+        mock_load = mock.Mock()
+        self.config.aug.load = mock_load
+
+        self.config.recovery_routine()
+        self.assertEqual(mock_load.call_count, 1)
+
+    def test_recovery_routine_error(self):
+        self.config.reverter.recovery_routine = mock.Mock(
+            side_effect=errors.ReverterError)
+
+        self.assertRaises(
+            errors.PluginError, self.config.recovery_routine)
+
+    def test_revert_challenge_config(self):
+        mock_load = mock.Mock()
+        self.config.aug.load = mock_load
+
+        self.config.revert_challenge_config()
+        self.assertEqual(mock_load.call_count, 1)
+
+    def test_revert_challenge_config_error(self):
+        self.config.reverter.revert_temporary_config = mock.Mock(
+            side_effect=errors.ReverterError)
+
+        self.assertRaises(
+            errors.PluginError, self.config.revert_challenge_config)
+
+    def test_rollback_checkpoints(self):
+        mock_load = mock.Mock()
+        self.config.aug.load = mock_load
+
+        self.config.rollback_checkpoints()
+        self.assertEqual(mock_load.call_count, 1)
+
+    def test_rollback_error(self):
+        self.config.reverter.rollback_checkpoints = mock.Mock(
+            side_effect=errors.ReverterError)
+        self.assertRaises(errors.PluginError, self.config.rollback_checkpoints)
+
+    def test_view_config_changes(self):
+        self.config.view_config_changes()
+
+    def test_view_config_changes_error(self):
+        self.config.reverter.view_config_changes = mock.Mock(
+            side_effect=errors.ReverterError)
+        self.assertRaises(errors.PluginError, self.config.view_config_changes)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

letsencrypt-apache/letsencrypt_apache/tests/complex_parsing_test.py

@@ -0,0 +1,127 @@
+"""Tests for letsencrypt_apache.parser."""
+import os
+import shutil
+import unittest
+
+from letsencrypt import errors
+
+from letsencrypt_apache.tests import util
+
+
+class ComplexParserTest(util.ParserTest):
+    """Apache Parser Test."""
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(ComplexParserTest, self).setUp(
+            "complex_parsing", "complex_parsing")
+
+        self.setup_variables()
+        # This needs to happen after due to setup_variables not being run
+        # until after
+        self.parser.init_modules()  # pylint: disable=protected-access
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir)
+        shutil.rmtree(self.config_dir)
+        shutil.rmtree(self.work_dir)
+
+    def setup_variables(self):
+        """Set up variables for parser."""
+        self.parser.variables.update(
+            {
+                "COMPLEX": "",
+                "tls_port": "1234",
+                "fnmatch_filename": "test_fnmatch.conf",
+                "tls_port_str": "1234"
+            }
+        )
+
+    def test_filter_args_num(self):
+        """Note: This may also fail do to Include conf-enabled/ syntax."""
+        matches = self.parser.find_dir("TestArgsDirective")
+
+        self.assertEqual(len(self.parser.filter_args_num(matches, 1)), 3)
+        self.assertEqual(len(self.parser.filter_args_num(matches, 2)), 2)
+        self.assertEqual(len(self.parser.filter_args_num(matches, 3)), 1)
+
+    def test_basic_variable_parsing(self):
+        matches = self.parser.find_dir("TestVariablePort")
+
+        self.assertEqual(len(matches), 1)
+        self.assertEqual(self.parser.get_arg(matches[0]), "1234")
+
+    def test_basic_variable_parsing_quotes(self):
+        matches = self.parser.find_dir("TestVariablePortStr")
+
+        self.assertEqual(len(matches), 1)
+        self.assertEqual(self.parser.get_arg(matches[0]), "1234")
+
+    def test_invalid_variable_parsing(self):
+        del self.parser.variables["tls_port"]
+
+        matches = self.parser.find_dir("TestVariablePort")
+        self.assertRaises(
+            errors.PluginError, self.parser.get_arg, matches[0])
+
+    def test_basic_ifdefine(self):
+        self.assertEqual(len(self.parser.find_dir("VAR_DIRECTIVE")), 2)
+        self.assertEqual(len(self.parser.find_dir("INVALID_VAR_DIRECTIVE")), 0)
+
+    def test_basic_ifmodule(self):
+        self.assertEqual(len(self.parser.find_dir("MOD_DIRECTIVE")), 2)
+        self.assertEqual(
+            len(self.parser.find_dir("INVALID_MOD_DIRECTIVE")), 0)
+
+    def test_nested(self):
+        self.assertEqual(len(self.parser.find_dir("NESTED_DIRECTIVE")), 3)
+        self.assertEqual(
+            len(self.parser.find_dir("INVALID_NESTED_DIRECTIVE")), 0)
+
+    def test_load_modules(self):
+        """If only first is found, there is bad variable parsing."""
+        self.assertTrue("status_module" in self.parser.modules)
+        self.assertTrue("mod_status.c" in self.parser.modules)
+
+        # This is in an IfDefine
+        self.assertTrue("ssl_module" in self.parser.modules)
+        self.assertTrue("mod_ssl.c" in self.parser.modules)
+
+    def verify_fnmatch(self, arg, hit=True):
+        """Test if Include was correctly parsed."""
+        from letsencrypt_apache import parser
+        self.parser.add_dir(parser.get_aug_path(self.parser.loc["default"]),
+                            "Include", [arg])
+        if hit:
+            self.assertTrue(self.parser.find_dir("FNMATCH_DIRECTIVE"))
+        else:
+            self.assertFalse(self.parser.find_dir("FNMATCH_DIRECTIVE"))
+
+    # NOTE: Only run one test per function otherwise you will have inf recursion
+    def test_include(self):
+        self.verify_fnmatch("test_fnmatch.?onf")
+
+    def test_include_complex(self):
+        self.verify_fnmatch("../complex_parsing/[te][te]st_*.?onf")
+
+    def test_include_fullpath(self):
+        self.verify_fnmatch(os.path.join(self.config_path, "test_fnmatch.conf"))
+
+    def test_include_fullpath_trailing_slash(self):
+        self.verify_fnmatch(self.config_path + "//")
+
+    def test_include_single_quotes(self):
+        self.verify_fnmatch("'" + self.config_path + "'")
+
+    def test_include_double_quotes(self):
+        self.verify_fnmatch('"' + self.config_path + '"')
+
+    def test_include_variable(self):
+        self.verify_fnmatch("../complex_parsing/${fnmatch_filename}")
+
+    def test_include_missing(self):
+        # This should miss
+        self.verify_fnmatch("test_*.onf", False)
+
+
+if __name__ == "__main__":
+    unittest.main()  # pragma: no cover

letsencrypt-apache/letsencrypt_apache/tests/configurator_test.py

@@ -1,7 +1,8 @@
+# pylint: disable=too-many-public-methods
 """Test for letsencrypt_apache.configurator."""
 import os
-import re
 import shutil
+import socket
 import unittest
 
 import mock
@@ -10,29 +11,23 @@ from acme import challenges
 
 from letsencrypt import achallenges
 from letsencrypt import errors
-from letsencrypt import le_util
-
-from letsencrypt.plugins import common
 
 from letsencrypt.tests import acme_util
 
 from letsencrypt_apache import configurator
-from letsencrypt_apache import parser
+from letsencrypt_apache import obj
 
 from letsencrypt_apache.tests import util
 
 
 class TwoVhost80Test(util.ApacheTest):
-    """Test two standard well configured HTTP vhosts."""
+    """Test two standard well-configured HTTP vhosts."""
 
-    def setUp(self):
+    def setUp(self):  # pylint: disable=arguments-differ
         super(TwoVhost80Test, self).setUp()
 
-        with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator."
-                        "mod_loaded") as mock_load:
-            mock_load.return_value = True
-            self.config = util.get_apache_configurator(
-                self.config_path, self.config_dir, self.work_dir)
+        self.config = util.get_apache_configurator(
+            self.config_path, self.config_dir, self.work_dir)
 
         self.vh_truth = util.get_vh_truth(
             self.temp_dir, "debian_apache_2_4/two_vhost_80")
@@ -42,16 +37,63 @@ class TwoVhost80Test(util.ApacheTest):
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
+    @mock.patch("letsencrypt_apache.configurator.le_util.exe_exists")
+    def test_prepare_no_install(self, mock_exe_exists):
+        mock_exe_exists.return_value = False
+        self.assertRaises(
+            errors.NoInstallationError, self.config.prepare)
+
+    @mock.patch("letsencrypt_apache.parser.ApacheParser")
+    @mock.patch("letsencrypt_apache.configurator.le_util.exe_exists")
+    def test_prepare_version(self, mock_exe_exists, _):
+        mock_exe_exists.return_value = True
+        self.config.version = None
+        self.config.config_test = mock.Mock()
+        self.config.get_version = mock.Mock(return_value=(1, 1))
+
+        self.assertRaises(
+            errors.NotSupportedError, self.config.prepare)
+
+    def test_add_parser_arguments(self):  # pylint: disable=no-self-use
+        from letsencrypt_apache.configurator import ApacheConfigurator
+        # Weak test..
+        ApacheConfigurator.add_parser_arguments(mock.MagicMock())
+
     def test_get_all_names(self):
         names = self.config.get_all_names()
         self.assertEqual(names, set(
             ["letsencrypt.demo", "encryption-example.demo", "ip-172-30-0-17"]))
 
+    @mock.patch("letsencrypt_apache.configurator.socket.gethostbyaddr")
+    def test_get_all_names_addrs(self, mock_gethost):
+        mock_gethost.side_effect = [("google.com", "", ""), socket.error]
+        vhost = obj.VirtualHost(
+            "fp", "ap",
+            set([obj.Addr(("8.8.8.8", "443")),
+                 obj.Addr(("zombo.com",)),
+                 obj.Addr(("192.168.1.2"))]),
+            True, False)
+        self.config.vhosts.append(vhost)
+
+        names = self.config.get_all_names()
+        self.assertEqual(len(names), 5)
+        self.assertTrue("zombo.com" in names)
+        self.assertTrue("google.com" in names)
+        self.assertTrue("letsencrypt.demo" in names)
+
+    def test_add_servernames_alias(self):
+        self.config.parser.add_dir(
+            self.vh_truth[2].path, "ServerAlias", ["*.le.co"])
+        self.config._add_servernames(self.vh_truth[2])  # pylint: disable=protected-access
+
+        self.assertEqual(
+            self.vh_truth[2].get_names(), set(["*.le.co", "ip-172-30-0-17"]))
+
     def test_get_virtual_hosts(self):
         """Make sure all vhosts are being properly found.
 
         .. note:: If test fails, only finding 1 Vhost... it is likely that
-            it is a problem with is_enabled.
+            it is a problem with is_enabled.  If finding only 3, likely is_ssl
 
         """
         vhs = self.config.get_virtual_hosts()
@@ -63,9 +105,77 @@ class TwoVhost80Test(util.ApacheTest):
                 if vhost == truth:
                     found += 1
                     break
+            else:
+                raise Exception("Missed: %s" % vhost)  # pragma: no cover
 
         self.assertEqual(found, 4)
 
+    @mock.patch("letsencrypt_apache.display_ops.select_vhost")
+    def test_choose_vhost_none_avail(self, mock_select):
+        mock_select.return_value = None
+        self.assertRaises(
+            errors.PluginError, self.config.choose_vhost, "none.com")
+
+    @mock.patch("letsencrypt_apache.display_ops.select_vhost")
+    def test_choose_vhost_select_vhost_ssl(self, mock_select):
+        mock_select.return_value = self.vh_truth[1]
+        self.assertEqual(
+            self.vh_truth[1], self.config.choose_vhost("none.com"))
+
+    @mock.patch("letsencrypt_apache.display_ops.select_vhost")
+    def test_choose_vhost_select_vhost_non_ssl(self, mock_select):
+        mock_select.return_value = self.vh_truth[0]
+        chosen_vhost = self.config.choose_vhost("none.com")
+        self.assertEqual(
+            self.vh_truth[0].get_names(), chosen_vhost.get_names())
+
+        # Make sure we go from HTTP -> HTTPS
+        self.assertFalse(self.vh_truth[0].ssl)
+        self.assertTrue(chosen_vhost.ssl)
+
+    @mock.patch("letsencrypt_apache.display_ops.select_vhost")
+    def test_choose_vhost_select_vhost_conflicting_non_ssl(self, mock_select):
+        mock_select.return_value = self.vh_truth[3]
+        conflicting_vhost = obj.VirtualHost(
+            "path", "aug_path", set([obj.Addr.fromstring("*:443")]), True, True)
+        self.config.vhosts.append(conflicting_vhost)
+
+        self.assertRaises(
+            errors.PluginError, self.config.choose_vhost, "none.com")
+
+    def test_find_best_vhost(self):
+        # pylint: disable=protected-access
+        self.assertEqual(
+            self.vh_truth[3], self.config._find_best_vhost("letsencrypt.demo"))
+        self.assertEqual(
+            self.vh_truth[0],
+            self.config._find_best_vhost("encryption-example.demo"))
+        self.assertTrue(
+            self.config._find_best_vhost("does-not-exist.com") is None)
+
+    def test_find_best_vhost_variety(self):
+        # pylint: disable=protected-access
+        ssl_vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]),
+            True, False)
+        self.config.vhosts.append(ssl_vh)
+        self.assertEqual(self.config._find_best_vhost("zombo.com"), ssl_vh)
+
+    def test_find_best_vhost_default(self):
+        # pylint: disable=protected-access
+        # Assume only the two default vhosts.
+        self.config.vhosts = [
+            vh for vh in self.config.vhosts
+            if vh.name not in ["letsencrypt.demo", "encryption-example.demo"]
+        ]
+
+        self.assertEqual(
+            self.config._find_best_vhost("example.demo"), self.vh_truth[2])
+
+    def test_non_default_vhosts(self):
+        # pylint: disable=protected-access
+        self.assertEqual(len(self.config._non_default_vhosts()), 3)
+
     def test_is_site_enabled(self):
         """Test if site is enabled.
 
@@ -80,7 +190,50 @@ class TwoVhost80Test(util.ApacheTest):
         self.assertTrue(self.config.is_site_enabled(self.vh_truth[2].filep))
         self.assertTrue(self.config.is_site_enabled(self.vh_truth[3].filep))
 
+    @mock.patch("letsencrypt.le_util.run_script")
+    @mock.patch("letsencrypt.le_util.exe_exists")
+    @mock.patch("letsencrypt_apache.parser.subprocess.Popen")
+    def test_enable_mod(self, mock_popen, mock_exe_exists, mock_run_script):
+        mock_popen().communicate.return_value = ("Define: DUMP_RUN_CFG", "")
+        mock_popen().returncode = 0
+        mock_exe_exists.return_value = True
+
+        self.config.enable_mod("ssl")
+        self.assertTrue("ssl_module" in self.config.parser.modules)
+        self.assertTrue("mod_ssl.c" in self.config.parser.modules)
+
+        self.assertTrue(mock_run_script.called)
+
+    def test_enable_mod_unsupported_dirs(self):
+        shutil.rmtree(os.path.join(self.config.parser.root, "mods-enabled"))
+        self.assertRaises(
+            errors.NotSupportedError, self.config.enable_mod, "ssl")
+
+    @mock.patch("letsencrypt.le_util.exe_exists")
+    def test_enable_mod_no_disable(self, mock_exe_exists):
+        mock_exe_exists.return_value = False
+        self.assertRaises(
+            errors.MisconfigurationError, self.config.enable_mod, "ssl")
+
+    def test_enable_site(self):
+        # Default 443 vhost
+        self.assertFalse(self.vh_truth[1].enabled)
+        self.config.enable_site(self.vh_truth[1])
+        self.assertTrue(self.vh_truth[1].enabled)
+
+        # Go again to make sure nothing fails
+        self.config.enable_site(self.vh_truth[1])
+
+    def test_enable_site_failure(self):
+        self.assertRaises(
+            errors.NotSupportedError,
+            self.config.enable_site,
+            obj.VirtualHost("asdf", "afsaf", set(), False, False))
+
     def test_deploy_cert(self):
+        self.config.parser.modules.add("ssl_module")
+        self.config.parser.modules.add("mod_ssl.c")
+
         # Get the default 443 vhost
         self.config.assoc["random.demo"] = self.vh_truth[1]
         self.config.deploy_cert(
@@ -88,15 +241,17 @@ class TwoVhost80Test(util.ApacheTest):
             "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
         self.config.save()
 
+        # Verify ssl_module was enabled.
+        self.assertTrue(self.vh_truth[1].enabled)
+        self.assertTrue("ssl_module" in self.config.parser.modules)
+
         loc_cert = self.config.parser.find_dir(
-            parser.case_i("sslcertificatefile"),
-            re.escape("example/cert.pem"), self.vh_truth[1].path)
+            "sslcertificatefile", "example/cert.pem", self.vh_truth[1].path)
         loc_key = self.config.parser.find_dir(
-            parser.case_i("sslcertificateKeyfile"),
-            re.escape("example/key.pem"), self.vh_truth[1].path)
+            "sslcertificateKeyfile", "example/key.pem", self.vh_truth[1].path)
         loc_chain = self.config.parser.find_dir(
-            parser.case_i("SSLCertificateChainFile"),
-            re.escape("example/cert_chain.pem"), self.vh_truth[1].path)
+            "SSLCertificateChainFile", "example/cert_chain.pem",
+            self.vh_truth[1].path)
 
         # Verify one directive was found in the correct file
         self.assertEqual(len(loc_cert), 1)
@@ -111,16 +266,60 @@ class TwoVhost80Test(util.ApacheTest):
         self.assertEqual(configurator.get_file_path(loc_chain[0]),
                          self.vh_truth[1].filep)
 
+        # One more time for chain directive setting
+        self.config.deploy_cert(
+            "random.demo",
+            "two/cert.pem", "two/key.pem", "two/cert_chain.pem")
+        self.assertTrue(self.config.parser.find_dir(
+            "SSLCertificateChainFile", "two/cert_chain.pem",
+            self.vh_truth[1].path))
+
+    def test_deploy_cert_invalid_vhost(self):
+        self.config.parser.modules.add("ssl_module")
+        mock_find = mock.MagicMock()
+        mock_find.return_value = []
+        self.config.parser.find_dir = mock_find
+
+        # Get the default 443 vhost
+        self.config.assoc["random.demo"] = self.vh_truth[1]
+        self.assertRaises(
+            errors.PluginError, self.config.deploy_cert, "random.demo",
+            "example/cert.pem", "example/key.pem", "example/cert_chain.pem")
+
     def test_is_name_vhost(self):
-        addr = common.Addr.fromstring("*:80")
+        addr = obj.Addr.fromstring("*:80")
         self.assertTrue(self.config.is_name_vhost(addr))
         self.config.version = (2, 2)
         self.assertFalse(self.config.is_name_vhost(addr))
 
     def test_add_name_vhost(self):
-        self.config.add_name_vhost("*:443")
+        self.config.add_name_vhost(obj.Addr.fromstring("*:443"))
+        self.config.add_name_vhost(obj.Addr.fromstring("*:80"))
         self.assertTrue(self.config.parser.find_dir(
-            "NameVirtualHost", re.escape("*:443")))
+            "NameVirtualHost", "*:443", exclude=False))
+        self.assertTrue(self.config.parser.find_dir(
+            "NameVirtualHost", "*:80"))
+
+    def test_prepare_server_https(self):
+        mock_enable = mock.Mock()
+        self.config.enable_mod = mock_enable
+
+        mock_find = mock.Mock()
+        mock_add_dir = mock.Mock()
+        mock_find.return_value = []
+
+        # This will test the Add listen
+        self.config.parser.find_dir = mock_find
+        self.config.parser.add_dir_to_ifmodssl = mock_add_dir
+
+        self.config.prepare_server_https("443")
+        self.assertEqual(mock_enable.call_args[1], {"temp": False})
+
+        self.config.prepare_server_https("8080", temp=True)
+        # Enable mod is temporary
+        self.assertEqual(mock_enable.call_args[1], {"temp": True})
+
+        self.assertEqual(mock_add_dir.call_count, 2)
 
     def test_make_vhost_ssl(self):
         ssl_vhost = self.config.make_vhost_ssl(self.vh_truth[0])
@@ -133,47 +332,58 @@ class TwoVhost80Test(util.ApacheTest):
         self.assertEqual(ssl_vhost.path,
                          "/files" + ssl_vhost.filep + "/IfModule/VirtualHost")
         self.assertEqual(len(ssl_vhost.addrs), 1)
-        self.assertEqual(set([common.Addr.fromstring("*:443")]), ssl_vhost.addrs)
-        self.assertEqual(ssl_vhost.names, set(["encryption-example.demo"]))
+        self.assertEqual(set([obj.Addr.fromstring("*:443")]), ssl_vhost.addrs)
+        self.assertEqual(ssl_vhost.name, "encryption-example.demo")
         self.assertTrue(ssl_vhost.ssl)
         self.assertFalse(ssl_vhost.enabled)
 
         self.assertTrue(self.config.parser.find_dir(
-            "SSLCertificateFile", None, ssl_vhost.path))
+            "SSLCertificateFile", None, ssl_vhost.path, False))
         self.assertTrue(self.config.parser.find_dir(
-            "SSLCertificateKeyFile", None, ssl_vhost.path))
-        self.assertTrue(self.config.parser.find_dir(
-            "Include", self.ssl_options, ssl_vhost.path))
+            "SSLCertificateKeyFile", None, ssl_vhost.path, False))
 
         self.assertEqual(self.config.is_name_vhost(self.vh_truth[0]),
                          self.config.is_name_vhost(ssl_vhost))
 
         self.assertEqual(len(self.config.vhosts), 5)
 
+    def test_make_vhost_ssl_extra_vhs(self):
+        self.config.aug.match = mock.Mock(return_value=["p1", "p2"])
+        self.assertRaises(
+            errors.PluginError, self.config.make_vhost_ssl, self.vh_truth[0])
+
+    def test_make_vhost_ssl_bad_write(self):
+        mock_open = mock.mock_open()
+        # This calls open
+        self.config.reverter.register_file_creation = mock.Mock()
+        mock_open.side_effect = IOError
+        with mock.patch("__builtin__.open", mock_open):
+            self.assertRaises(
+                errors.PluginError,
+                self.config.make_vhost_ssl, self.vh_truth[0])
+
+    def test_get_ssl_vhost_path(self):
+        # pylint: disable=protected-access
+        self.assertTrue(
+            self.config._get_ssl_vhost_path("example_path").endswith(".conf"))
+
+    def test_add_name_vhost_if_necessary(self):
+        # pylint: disable=protected-access
+        self.config.save = mock.Mock()
+        self.config.version = (2, 2)
+        self.config._add_name_vhost_if_necessary(self.vh_truth[0])
+        self.assertTrue(self.config.save.called)
+
     @mock.patch("letsencrypt_apache.configurator.dvsni.ApacheDvsni.perform")
     @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart")
     def test_perform(self, mock_restart, mock_dvsni_perform):
         # Only tests functionality specific to configurator.perform
         # Note: As more challenges are offered this will have to be expanded
-        auth_key = le_util.Key(self.rsa256_file, self.rsa256_pem)
-        achall1 = achallenges.DVSNI(
-            challb=acme_util.chall_to_challb(
-                challenges.DVSNI(
-                    r="jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q",
-                    nonce="37bc5eb75d3e00a19b4f6355845e5a18"),
-                "pending"),
-            domain="encryption-example.demo", key=auth_key)
-        achall2 = achallenges.DVSNI(
-            challb=acme_util.chall_to_challb(
-                challenges.DVSNI(
-                    r="uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU",
-                    nonce="59ed014cac95f77057b1d7a1b2c596ba"),
-                "pending"),
-            domain="letsencrypt.demo", key=auth_key)
+        account_key, achall1, achall2 = self.get_achalls()
 
         dvsni_ret_val = [
-            challenges.DVSNIResponse(s="randomS1"),
-            challenges.DVSNIResponse(s="randomS2"),
+            achall1.gen_response(account_key),
+            achall2.gen_response(account_key),
         ]
 
         mock_dvsni_perform.return_value = dvsni_ret_val
@@ -184,27 +394,228 @@ class TwoVhost80Test(util.ApacheTest):
 
         self.assertEqual(mock_restart.call_count, 1)
 
-    @mock.patch("letsencrypt_apache.configurator.subprocess.Popen")
-    def test_get_version(self, mock_popen):
-        mock_popen().communicate.return_value = (
+    @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart")
+    def test_cleanup(self, mock_restart):
+        _, achall1, achall2 = self.get_achalls()
+
+        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
+        self.config._chall_out.add(achall2)  # pylint: disable=protected-access
+
+        self.config.cleanup([achall1])
+        self.assertFalse(mock_restart.called)
+
+        self.config.cleanup([achall2])
+        self.assertTrue(mock_restart.called)
+
+    @mock.patch("letsencrypt_apache.configurator.ApacheConfigurator.restart")
+    def test_cleanup_no_errors(self, mock_restart):
+        _, achall1, achall2 = self.get_achalls()
+
+        self.config._chall_out.add(achall1)  # pylint: disable=protected-access
+
+        self.config.cleanup([achall2])
+        self.assertFalse(mock_restart.called)
+
+        self.config.cleanup([achall1, achall2])
+        self.assertTrue(mock_restart.called)
+
+    @mock.patch("letsencrypt.le_util.run_script")
+    def test_get_version(self, mock_script):
+        mock_script.return_value = (
             "Server Version: Apache/2.4.2 (Debian)", "")
         self.assertEqual(self.config.get_version(), (2, 4, 2))
 
-        mock_popen().communicate.return_value = (
+        mock_script.return_value = (
             "Server Version: Apache/2 (Linux)", "")
         self.assertEqual(self.config.get_version(), (2,))
 
-        mock_popen().communicate.return_value = (
+        mock_script.return_value = (
             "Server Version: Apache (Debian)", "")
         self.assertRaises(errors.PluginError, self.config.get_version)
 
-        mock_popen().communicate.return_value = (
+        mock_script.return_value = (
             "Server Version: Apache/2.3{0} Apache/2.4.7".format(os.linesep), "")
         self.assertRaises(errors.PluginError, self.config.get_version)
 
-        mock_popen.side_effect = OSError("Can't find program")
+        mock_script.side_effect = errors.SubprocessError("Can't find program")
         self.assertRaises(errors.PluginError, self.config.get_version)
 
+    @mock.patch("letsencrypt_apache.configurator.subprocess.Popen")
+    def test_restart(self, mock_popen):
+        """These will be changed soon enough with reload."""
+        mock_popen().returncode = 0
+        mock_popen().communicate.return_value = ("", "")
+
+        self.config.restart()
+
+    @mock.patch("letsencrypt_apache.configurator.subprocess.Popen")
+    def test_restart_bad_process(self, mock_popen):
+        mock_popen.side_effect = OSError
+
+        self.assertRaises(errors.MisconfigurationError, self.config.restart)
+
+    @mock.patch("letsencrypt_apache.configurator.subprocess.Popen")
+    def test_restart_failure(self, mock_popen):
+        mock_popen().communicate.return_value = ("", "")
+        mock_popen().returncode = 1
+
+        self.assertRaises(errors.MisconfigurationError, self.config.restart)
+
+    @mock.patch("letsencrypt.le_util.run_script")
+    def test_config_test(self, _):
+        self.config.config_test()
+
+    @mock.patch("letsencrypt.le_util.run_script")
+    def test_config_test_bad_process(self, mock_run_script):
+        mock_run_script.side_effect = errors.SubprocessError
+
+        self.assertRaises(errors.MisconfigurationError, self.config.config_test)
+
+    def test_get_all_certs_keys(self):
+        c_k = self.config.get_all_certs_keys()
+
+        self.assertEqual(len(c_k), 1)
+        cert, key, path = next(iter(c_k))
+        self.assertTrue("cert" in cert)
+        self.assertTrue("key" in key)
+        self.assertTrue("default-ssl.conf" in path)
+
+    def test_get_all_certs_keys_malformed_conf(self):
+        self.config.parser.find_dir = mock.Mock(side_effect=[["path"], []])
+        c_k = self.config.get_all_certs_keys()
+
+        self.assertFalse(c_k)
+
+    def test_more_info(self):
+        self.assertTrue(self.config.more_info())
+
+    def test_get_chall_pref(self):
+        self.assertTrue(isinstance(self.config.get_chall_pref(""), list))
+
+    def test_temp_install(self):
+        from letsencrypt_apache.configurator import temp_install
+        path = os.path.join(self.work_dir, "test_it")
+        temp_install(path)
+        self.assertTrue(os.path.isfile(path))
+
+    # TEST ENHANCEMENTS
+    def test_supported_enhancements(self):
+        self.assertTrue(isinstance(self.config.supported_enhancements(), list))
+
+    def test_enhance_unknown_enhancement(self):
+        self.assertRaises(
+            errors.PluginError,
+            self.config.enhance, "letsencrypt.demo", "unknown_enhancement")
+
+    @mock.patch("letsencrypt.le_util.run_script")
+    @mock.patch("letsencrypt.le_util.exe_exists")
+    def test_redirect_well_formed_http(self, mock_exe, _):
+        self.config.parser.update_runtime_variables = mock.Mock()
+        mock_exe.return_value = True
+        # This will create an ssl vhost for letsencrypt.demo
+        self.config.enhance("letsencrypt.demo", "redirect")
+
+        # These are not immediately available in find_dir even with save() and
+        # load(). They must be found in sites-available
+        rw_engine = self.config.parser.find_dir(
+            "RewriteEngine", "on", self.vh_truth[3].path)
+        rw_rule = self.config.parser.find_dir(
+            "RewriteRule", None, self.vh_truth[3].path)
+
+        self.assertEqual(len(rw_engine), 1)
+        # three args to rw_rule
+        self.assertEqual(len(rw_rule), 3)
+
+        self.assertTrue(rw_engine[0].startswith(self.vh_truth[3].path))
+        self.assertTrue(rw_rule[0].startswith(self.vh_truth[3].path))
+
+        self.assertTrue("rewrite_module" in self.config.parser.modules)
+
+    def test_redirect_with_conflict(self):
+        self.config.parser.modules.add("rewrite_module")
+        ssl_vh = obj.VirtualHost(
+            "fp", "ap", set([obj.Addr(("*", "443")), obj.Addr(("zombo.com",))]),
+            True, False)
+        # No names ^ this guy should conflict.
+
+        # pylint: disable=protected-access
+        self.assertRaises(
+            errors.PluginError, self.config._enable_redirect, ssl_vh, "")
+
+    def test_redirect_twice(self):
+        # Skip the enable mod
+        self.config.parser.modules.add("rewrite_module")
+        self.config.enhance("encryption-example.demo", "redirect")
+        self.assertRaises(
+            errors.PluginError,
+            self.config.enhance, "encryption-example.demo", "redirect")
+
+    def test_unknown_rewrite(self):
+        # Skip the enable mod
+        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.add_dir(
+            self.vh_truth[3].path, "RewriteRule", ["Unknown"])
+        self.config.save()
+        self.assertRaises(
+            errors.PluginError,
+            self.config.enhance, "letsencrypt.demo", "redirect")
+
+    def test_unknown_rewrite2(self):
+        # Skip the enable mod
+        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.add_dir(
+            self.vh_truth[3].path, "RewriteRule", ["Unknown", "2", "3"])
+        self.config.save()
+        self.assertRaises(
+            errors.PluginError,
+            self.config.enhance, "letsencrypt.demo", "redirect")
+
+    def test_unknown_redirect(self):
+        # Skip the enable mod
+        self.config.parser.modules.add("rewrite_module")
+        self.config.parser.add_dir(
+            self.vh_truth[3].path, "Redirect", ["Unknown"])
+        self.config.save()
+        self.assertRaises(
+            errors.PluginError,
+            self.config.enhance, "letsencrypt.demo", "redirect")
+
+    def test_create_own_redirect(self):
+        self.config.parser.modules.add("rewrite_module")
+        # For full testing... give names...
+        self.vh_truth[1].name = "default.com"
+        self.vh_truth[1].aliases = set(["yes.default.com"])
+
+        self.config._enable_redirect(self.vh_truth[1], "")  # pylint: disable=protected-access
+        self.assertEqual(len(self.config.vhosts), 5)
+
+    def get_achalls(self):
+        """Return testing achallenges."""
+        account_key = self.rsa512jwk
+        achall1 = achallenges.DVSNI(
+            challb=acme_util.chall_to_challb(
+                challenges.DVSNI(
+                    token="jIq_Xy1mXGN37tb4L6Xj_es58fW571ZNyXekdZzhh7Q"),
+                "pending"),
+            domain="encryption-example.demo", account_key=account_key)
+        achall2 = achallenges.DVSNI(
+            challb=acme_util.chall_to_challb(
+                challenges.DVSNI(
+                    token="uqnaPzxtrndteOqtrXb0Asl5gOJfWAnnx6QJyvcmlDU"),
+                "pending"),
+            domain="letsencrypt.demo", account_key=account_key)
+
+        return account_key, achall1, achall2
+
+    def test_make_addrs_sni_ready(self):
+        self.config.version = (2, 2)
+        self.config.make_addrs_sni_ready(
+            set([obj.Addr.fromstring("*:443"), obj.Addr.fromstring("*:80")]))
+        self.assertTrue(self.config.parser.find_dir(
+            "NameVirtualHost", "*:80", exclude=False))
+        self.assertTrue(self.config.parser.find_dir(
+            "NameVirtualHost", "*:443", exclude=False))
+
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

letsencrypt-apache/letsencrypt_apache/tests/display_ops_test.py

@@ -5,10 +5,12 @@ import unittest
 import mock
 import zope.component
 
-from letsencrypt_apache.tests import util
-
 from letsencrypt.display import util as display_util
 
+from letsencrypt_apache import obj
+
+from letsencrypt_apache.tests import util
+
 
 class SelectVhostTest(unittest.TestCase):
     """Tests for letsencrypt_apache.display_ops.select_vhost."""
@@ -53,6 +55,18 @@ class SelectVhostTest(unittest.TestCase):
 
         self.assertEqual(mock_logger.debug.call_count, 1)
 
+    @mock.patch("letsencrypt_apache.display_ops.zope.component.getUtility")
+    def test_multiple_names(self, mock_util):
+        mock_util().menu.return_value = (display_util.OK, 4)
+
+        self.vhosts.append(
+            obj.VirtualHost(
+                "path", "aug_path", set([obj.Addr.fromstring("*:80")]),
+                False, False,
+                "wildcard.com", set(["*.wildcard.com"])))
+
+        self.assertEqual(self.vhosts[4], self._call(self.vhosts))
+
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

letsencrypt-apache/letsencrypt_apache/tests/dvsni_test.py

@@ -4,27 +4,24 @@ import shutil
 
 import mock
 
-from acme import challenges
-
-from letsencrypt.plugins import common
 from letsencrypt.plugins import common_test
 
+from letsencrypt_apache import obj
 from letsencrypt_apache.tests import util
 
 
 class DvsniPerformTest(util.ApacheTest):
     """Test the ApacheDVSNI challenge."""
 
+    auth_key = common_test.DvsniTest.auth_key
     achalls = common_test.DvsniTest.achalls
 
-    def setUp(self):
+    def setUp(self):  # pylint: disable=arguments-differ
         super(DvsniPerformTest, self).setUp()
 
-        with mock.patch("letsencrypt_apache.configurator.ApacheConfigurator."
-                        "mod_loaded") as mock_load:
-            mock_load.return_value = True
-            config = util.get_apache_configurator(
-                self.config_path, self.config_dir, self.work_dir)
+        config = util.get_apache_configurator(
+            self.config_path, self.config_dir, self.work_dir)
+        config.config.dvsni_port = 443
 
         from letsencrypt_apache import dvsni
         self.sni = dvsni.ApacheDvsni(config)
@@ -38,37 +35,50 @@ class DvsniPerformTest(util.ApacheTest):
         resp = self.sni.perform()
         self.assertEqual(len(resp), 0)
 
-    def test_perform1(self):
+    @mock.patch("letsencrypt.le_util.exe_exists")
+    @mock.patch("letsencrypt.le_util.run_script")
+    def test_perform1(self, _, mock_exists):
+        mock_register = mock.Mock()
+        self.sni.configurator.reverter.register_undo_command = mock_register
+
+        mock_exists.return_value = True
+        self.sni.configurator.parser.update_runtime_variables = mock.Mock()
+
         achall = self.achalls[0]
         self.sni.add_chall(achall)
-        mock_setup_cert = mock.MagicMock(
-            return_value=challenges.DVSNIResponse(s="randomS1"))
+        response = self.achalls[0].gen_response(self.auth_key)
+        mock_setup_cert = mock.MagicMock(return_value=response)
         # pylint: disable=protected-access
         self.sni._setup_challenge_cert = mock_setup_cert
 
         responses = self.sni.perform()
 
+        # Make sure that register_undo_command was called into temp directory.
+        self.assertEqual(True, mock_register.call_args[0][0])
+
         mock_setup_cert.assert_called_once_with(achall)
 
         # Check to make sure challenge config path is included in apache config.
         self.assertEqual(
             len(self.sni.configurator.parser.find_dir(
-                "Include", self.sni.challenge_conf)),
-            1)
+                "Include", self.sni.challenge_conf)), 1)
         self.assertEqual(len(responses), 1)
-        self.assertEqual(responses[0].s, "randomS1")
+        self.assertEqual(responses[0], response)
 
     def test_perform2(self):
+        # Avoid load module
+        self.sni.configurator.parser.modules.add("ssl_module")
+
+        acme_responses = []
         for achall in self.achalls:
             self.sni.add_chall(achall)
+            acme_responses.append(achall.gen_response(self.auth_key))
 
-        mock_setup_cert = mock.MagicMock(side_effect=[
-            challenges.DVSNIResponse(s="randomS0"),
-            challenges.DVSNIResponse(s="randomS1")])
+        mock_setup_cert = mock.MagicMock(side_effect=acme_responses)
         # pylint: disable=protected-access
         self.sni._setup_challenge_cert = mock_setup_cert
 
-        responses = self.sni.perform()
+        sni_responses = self.sni.perform()
 
         self.assertEqual(mock_setup_cert.call_count, 2)
 
@@ -82,20 +92,18 @@ class DvsniPerformTest(util.ApacheTest):
             len(self.sni.configurator.parser.find_dir(
                 "Include", self.sni.challenge_conf)),
             1)
-        self.assertEqual(len(responses), 2)
+        self.assertEqual(len(sni_responses), 2)
         for i in xrange(2):
-            self.assertEqual(responses[i].s, "randomS%d" % i)
+            self.assertEqual(sni_responses[i], acme_responses[i])
 
     def test_mod_config(self):
+        z_domains = []
         for achall in self.achalls:
             self.sni.add_chall(achall)
-        v_addr1 = [common.Addr(("1.2.3.4", "443")),
-                   common.Addr(("5.6.7.8", "443"))]
-        v_addr2 = [common.Addr(("127.0.0.1", "443"))]
-        ll_addr = []
-        ll_addr.append(v_addr1)
-        ll_addr.append(v_addr2)
-        self.sni._mod_config(ll_addr)  # pylint: disable=protected-access
+            z_domain = achall.gen_response(self.auth_key).z_domain
+            z_domains.append(set([z_domain]))
+
+        self.sni._mod_config()  # pylint: disable=protected-access
         self.sni.configurator.save()
 
         self.sni.configurator.parser.find_dir(
@@ -109,15 +117,20 @@ class DvsniPerformTest(util.ApacheTest):
             vhs.append(self.sni.configurator._create_vhost(match))
         self.assertEqual(len(vhs), 2)
         for vhost in vhs:
-            if vhost.addrs == set(v_addr1):
-                self.assertEqual(
-                    vhost.names,
-                    set([self.achalls[0].nonce_domain]))
-            else:
-                self.assertEqual(vhost.addrs, set(v_addr2))
-                self.assertEqual(
-                    vhost.names,
-                    set([self.achalls[1].nonce_domain]))
+            self.assertEqual(vhost.addrs, set([obj.Addr.fromstring("*:443")]))
+            names = vhost.get_names()
+            self.assertTrue(names in z_domains)
+
+    def test_get_dvsni_addrs_default(self):
+        self.sni.configurator.choose_vhost = mock.Mock(
+            return_value=obj.VirtualHost(
+                "path", "aug_path", set([obj.Addr.fromstring("_default_:443")]),
+                False, False)
+        )
+
+        self.assertEqual(
+            set([obj.Addr.fromstring("*:443")]),
+            self.sni.get_dvsni_addrs(self.achalls[0]))
 
 
 if __name__ == "__main__":

letsencrypt-apache/letsencrypt_apache/tests/obj_test.py

@@ -1,27 +1,135 @@
 """Tests for letsencrypt_apache.obj."""
 import unittest
 
-from letsencrypt.plugins import common
-
 
 class VirtualHostTest(unittest.TestCase):
     """Test the VirtualHost class."""
 
     def setUp(self):
+        from letsencrypt_apache.obj import Addr
         from letsencrypt_apache.obj import VirtualHost
+
+        self.addr1 = Addr.fromstring("127.0.0.1")
+        self.addr2 = Addr.fromstring("127.0.0.1:443")
+        self.addr_default = Addr.fromstring("_default_:443")
+
         self.vhost1 = VirtualHost(
-            "filep", "vh_path",
-            set([common.Addr.fromstring("localhost")]), False, False)
+            "filep", "vh_path", set([self.addr1]), False, False, "localhost")
+
+        self.vhost1b = VirtualHost(
+            "filep", "vh_path", set([self.addr1]), False, False, "localhost")
+
+        self.vhost2 = VirtualHost(
+            "fp", "vhp", set([self.addr2]), False, False, "localhost")
 
     def test_eq(self):
-        from letsencrypt_apache.obj import VirtualHost
-        vhost1b = VirtualHost(
-            "filep", "vh_path",
-            set([common.Addr.fromstring("localhost")]), False, False)
+        self.assertTrue(self.vhost1b == self.vhost1)
+        self.assertFalse(self.vhost1 == self.vhost2)
+        self.assertEqual(str(self.vhost1b), str(self.vhost1))
+        self.assertFalse(self.vhost1b == 1234)
 
-        self.assertEqual(vhost1b, self.vhost1)
-        self.assertEqual(str(vhost1b), str(self.vhost1))
-        self.assertFalse(vhost1b == 1234)
+    def test_ne(self):
+        self.assertTrue(self.vhost1 != self.vhost2)
+        self.assertFalse(self.vhost1 != self.vhost1b)
+
+    def test_conflicts(self):
+        from letsencrypt_apache.obj import Addr
+        from letsencrypt_apache.obj import VirtualHost
+
+        complex_vh = VirtualHost(
+            "fp", "vhp",
+            set([Addr.fromstring("*:443"), Addr.fromstring("1.2.3.4:443")]),
+            False, False)
+        self.assertTrue(complex_vh.conflicts([self.addr1]))
+        self.assertTrue(complex_vh.conflicts([self.addr2]))
+        self.assertFalse(complex_vh.conflicts([self.addr_default]))
+
+        self.assertTrue(self.vhost1.conflicts([self.addr2]))
+        self.assertFalse(self.vhost1.conflicts([self.addr_default]))
+
+        self.assertFalse(self.vhost2.conflicts([self.addr1, self.addr_default]))
+
+    def test_same_server(self):
+        from letsencrypt_apache.obj import VirtualHost
+        no_name1 = VirtualHost(
+            "fp", "vhp", set([self.addr1]), False, False, None)
+        no_name2 = VirtualHost(
+            "fp", "vhp", set([self.addr2]), False, False, None)
+        no_name3 = VirtualHost(
+            "fp", "vhp", set([self.addr_default]),
+            False, False, None)
+        no_name4 = VirtualHost(
+            "fp", "vhp", set([self.addr2, self.addr_default]),
+            False, False, None)
+
+        self.assertTrue(self.vhost1.same_server(self.vhost2))
+        self.assertTrue(no_name1.same_server(no_name2))
+
+        self.assertFalse(self.vhost1.same_server(no_name1))
+        self.assertFalse(no_name1.same_server(no_name3))
+        self.assertFalse(no_name1.same_server(no_name4))
+
+
+class AddrTest(unittest.TestCase):
+    """Test obj.Addr."""
+    def setUp(self):
+        from letsencrypt_apache.obj import Addr
+        self.addr = Addr.fromstring("*:443")
+
+        self.addr1 = Addr.fromstring("127.0.0.1")
+        self.addr2 = Addr.fromstring("127.0.0.1:*")
+
+        self.addr_defined = Addr.fromstring("127.0.0.1:443")
+        self.addr_default = Addr.fromstring("_default_:443")
+
+    def test_wildcard(self):
+        self.assertFalse(self.addr.is_wildcard())
+        self.assertTrue(self.addr1.is_wildcard())
+        self.assertTrue(self.addr2.is_wildcard())
+
+    def test_get_sni_addr(self):
+        from letsencrypt_apache.obj import Addr
+        self.assertEqual(
+            self.addr.get_sni_addr("443"), Addr.fromstring("*:443"))
+        self.assertEqual(
+            self.addr.get_sni_addr("225"), Addr.fromstring("*:225"))
+        self.assertEqual(
+            self.addr1.get_sni_addr("443"), Addr.fromstring("127.0.0.1"))
+
+    def test_conflicts(self):
+        # Note: Defined IP is more important than defined port in match
+        self.assertTrue(self.addr.conflicts(self.addr1))
+        self.assertTrue(self.addr.conflicts(self.addr2))
+        self.assertTrue(self.addr.conflicts(self.addr_defined))
+        self.assertFalse(self.addr.conflicts(self.addr_default))
+
+        self.assertFalse(self.addr1.conflicts(self.addr))
+        self.assertTrue(self.addr1.conflicts(self.addr_defined))
+        self.assertFalse(self.addr1.conflicts(self.addr_default))
+
+        self.assertFalse(self.addr_defined.conflicts(self.addr1))
+        self.assertFalse(self.addr_defined.conflicts(self.addr2))
+        self.assertFalse(self.addr_defined.conflicts(self.addr))
+        self.assertFalse(self.addr_defined.conflicts(self.addr_default))
+
+        self.assertTrue(self.addr_default.conflicts(self.addr))
+        self.assertTrue(self.addr_default.conflicts(self.addr1))
+        self.assertTrue(self.addr_default.conflicts(self.addr_defined))
+
+        # Self test
+        self.assertTrue(self.addr.conflicts(self.addr))
+        self.assertTrue(self.addr1.conflicts(self.addr1))
+        # This is a tricky one...
+        self.assertTrue(self.addr1.conflicts(self.addr2))
+
+    def test_equal(self):
+        self.assertTrue(self.addr1 == self.addr2)
+        self.assertFalse(self.addr == self.addr1)
+        self.assertFalse(self.addr == 123)
+
+    def test_not_equal(self):
+        self.assertFalse(self.addr1 != self.addr2)
+        self.assertTrue(self.addr != self.addr1)
 
 
 if __name__ == "__main__":

letsencrypt-apache/letsencrypt_apache/tests/parser_test.py

@@ -1,52 +1,32 @@
 """Tests for letsencrypt_apache.parser."""
 import os
 import shutil
-import sys
 import unittest
 
 import augeas
 import mock
-import zope.component
 
 from letsencrypt import errors
-from letsencrypt.display import util as display_util
 
 from letsencrypt_apache.tests import util
 
 
-class ApacheParserTest(util.ApacheTest):
+class BasicParserTest(util.ParserTest):
     """Apache Parser Test."""
 
-    def setUp(self):
-        super(ApacheParserTest, self).setUp()
-
-        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))
-
-        from letsencrypt_apache.parser import ApacheParser
-        self.aug = augeas.Augeas(flags=augeas.Augeas.NONE)
-        self.parser = ApacheParser(self.aug, self.config_path, self.ssl_options)
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(BasicParserTest, self).setUp()
 
     def tearDown(self):
         shutil.rmtree(self.temp_dir)
         shutil.rmtree(self.config_dir)
         shutil.rmtree(self.work_dir)
 
-    def test_root_normalized(self):
-        from letsencrypt_apache.parser import ApacheParser
-        path = os.path.join(self.temp_dir, "debian_apache_2_4/////"
-                            "two_vhost_80/../two_vhost_80/apache2")
-        parser = ApacheParser(self.aug, path, None)
-        self.assertEqual(parser.root, self.config_path)
-
-    def test_root_absolute(self):
-        from letsencrypt_apache.parser import ApacheParser
-        parser = ApacheParser(self.aug, os.path.relpath(self.config_path), None)
-        self.assertEqual(parser.root, self.config_path)
-
-    def test_root_no_trailing_slash(self):
-        from letsencrypt_apache.parser import ApacheParser
-        parser = ApacheParser(self.aug, self.config_path + os.path.sep, None)
-        self.assertEqual(parser.root, self.config_path)
+    def test_find_config_root_no_root(self):
+        # pylint: disable=protected-access
+        os.remove(self.parser.loc["root"])
+        self.assertRaises(
+            errors.NoInstallationError, self.parser._find_config_root)
 
     def test_parse_file(self):
         """Test parse_file.
@@ -67,11 +47,11 @@ class ApacheParserTest(util.ApacheTest):
         self.assertTrue(matches)
 
     def test_find_dir(self):
-        from letsencrypt_apache.parser import case_i
-        test = self.parser.find_dir(case_i("Listen"), "443")
+        test = self.parser.find_dir("Listen", "80")
         # This will only look in enabled hosts
-        test2 = self.parser.find_dir(case_i("documentroot"))
-        self.assertEqual(len(test), 2)
+        test2 = self.parser.find_dir("documentroot")
+
+        self.assertEqual(len(test), 1)
         self.assertEqual(len(test2), 3)
 
     def test_add_dir(self):
@@ -93,15 +73,32 @@ class ApacheParserTest(util.ApacheTest):
 
         """
         from letsencrypt_apache.parser import get_aug_path
+        # This makes sure that find_dir will work
+        self.parser.modules.add("mod_ssl.c")
+
         self.parser.add_dir_to_ifmodssl(
             get_aug_path(self.parser.loc["default"]),
-            "FakeDirective", "123")
+            "FakeDirective", ["123"])
 
         matches = self.parser.find_dir("FakeDirective", "123")
 
         self.assertEqual(len(matches), 1)
         self.assertTrue("IfModule" in matches[0])
 
+    def test_add_dir_to_ifmodssl_multiple(self):
+        from letsencrypt_apache.parser import get_aug_path
+        # This makes sure that find_dir will work
+        self.parser.modules.add("mod_ssl.c")
+
+        self.parser.add_dir_to_ifmodssl(
+            get_aug_path(self.parser.loc["default"]),
+            "FakeDirective", ["123", "456", "789"])
+
+        matches = self.parser.find_dir("FakeDirective")
+
+        self.assertEqual(len(matches), 3)
+        self.assertTrue("IfModule" in matches[0])
+
     def test_get_aug_path(self):
         from letsencrypt_apache.parser import get_aug_path
         self.assertEqual("/files/etc/apache", get_aug_path("/etc/apache"))
@@ -109,20 +106,114 @@ class ApacheParserTest(util.ApacheTest):
     def test_set_locations(self):
         with mock.patch("letsencrypt_apache.parser.os.path") as mock_path:
 
-            mock_path.isfile.return_value = False
-
-            # pylint: disable=protected-access
-            self.assertRaises(errors.PluginError,
-                              self.parser._set_locations, self.ssl_options)
-
             mock_path.isfile.side_effect = [True, False, False]
 
             # pylint: disable=protected-access
-            results = self.parser._set_locations(self.ssl_options)
+            results = self.parser._set_locations()
 
             self.assertEqual(results["default"], results["listen"])
             self.assertEqual(results["default"], results["name"])
 
+    def test_set_user_config_file(self):
+        # pylint: disable=protected-access
+        path = os.path.join(self.parser.root, "httpd.conf")
+        open(path, 'w').close()
+        self.parser.add_dir(self.parser.loc["default"], "Include", "httpd.conf")
+
+        self.assertEqual(
+            path, self.parser._set_user_config_file())
+
+    @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg")
+    def test_update_runtime_variables(self, mock_cfg):
+        mock_cfg.return_value = (
+            'ServerRoot: "/etc/apache2"\n'
+            'Main DocumentRoot: "/var/www"\n'
+            'Main ErrorLog: "/var/log/apache2/error.log"\n'
+            'Mutex ssl-stapling: using_defaults\n'
+            'Mutex ssl-cache: using_defaults\n'
+            'Mutex default: dir="/var/lock/apache2" mechanism=fcntl\n'
+            'Mutex watchdog-callback: using_defaults\n'
+            'PidFile: "/var/run/apache2/apache2.pid"\n'
+            'Define: TEST\n'
+            'Define: DUMP_RUN_CFG\n'
+            'Define: U_MICH\n'
+            'Define: TLS=443\n'
+            'Define: example_path=Documents/path\n'
+            'User: name="www-data" id=33 not_used\n'
+            'Group: name="www-data" id=33 not_used\n'
+        )
+        expected_vars = {"TEST": "", "U_MICH": "", "TLS": "443",
+                         "example_path": "Documents/path"}
+
+        self.parser.update_runtime_variables("ctl")
+        self.assertEqual(self.parser.variables, expected_vars)
+
+    @mock.patch("letsencrypt_apache.parser.ApacheParser._get_runtime_cfg")
+    def test_update_runtime_vars_bad_output(self, mock_cfg):
+        mock_cfg.return_value = "Define: TLS=443=24"
+        self.assertRaises(
+            errors.PluginError, self.parser.update_runtime_variables, "ctl")
+
+        mock_cfg.return_value = "Define: DUMP_RUN_CFG\nDefine: TLS=443=24"
+        self.assertRaises(
+            errors.PluginError, self.parser.update_runtime_variables, "ctl")
+
+    @mock.patch("letsencrypt_apache.parser.subprocess.Popen")
+    def test_update_runtime_vars_bad_ctl(self, mock_popen):
+        mock_popen.side_effect = OSError
+        self.assertRaises(
+            errors.MisconfigurationError,
+            self.parser.update_runtime_variables, "ctl")
+
+    @mock.patch("letsencrypt_apache.parser.subprocess.Popen")
+    def test_update_runtime_vars_bad_exit(self, mock_popen):
+        mock_popen().communicate.return_value = ("", "")
+        mock_popen.returncode = -1
+        self.assertRaises(
+            errors.MisconfigurationError,
+            self.parser.update_runtime_variables, "ctl")
+
+
+class ParserInitTest(util.ApacheTest):
+    def setUp(self):  # pylint: disable=arguments-differ
+        super(ParserInitTest, self).setUp()
+        self.aug = augeas.Augeas(
+            flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir)
+        shutil.rmtree(self.config_dir)
+        shutil.rmtree(self.work_dir)
+
+    def test_root_normalized(self):
+        from letsencrypt_apache.parser import ApacheParser
+
+        with mock.patch("letsencrypt_apache.parser.ApacheParser."
+                        "update_runtime_variables"):
+            path = os.path.join(
+                self.temp_dir,
+                "debian_apache_2_4/////two_vhost_80/../two_vhost_80/apache2")
+            parser = ApacheParser(self.aug, path, "dummy_ctl")
+
+        self.assertEqual(parser.root, self.config_path)
+
+    def test_root_absolute(self):
+        from letsencrypt_apache.parser import ApacheParser
+        with mock.patch("letsencrypt_apache.parser.ApacheParser."
+                        "update_runtime_variables"):
+            parser = ApacheParser(
+                self.aug, os.path.relpath(self.config_path), "dummy_ctl")
+
+        self.assertEqual(parser.root, self.config_path)
+
+    def test_root_no_trailing_slash(self):
+        from letsencrypt_apache.parser import ApacheParser
+        with mock.patch("letsencrypt_apache.parser.ApacheParser."
+                        "update_runtime_variables"):
+            parser = ApacheParser(
+                self.aug, self.config_path + os.path.sep, "dummy_ctl")
+        self.assertEqual(parser.root, self.config_path)
+
 
 if __name__ == "__main__":
     unittest.main()  # pragma: no cover

letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/apache2.conf

@@ -0,0 +1,55 @@
+# Global configuration
+
+PidFile ${APACHE_PID_FILE}
+
+#
+# Timeout: The number of seconds before receives and sends time out.
+#
+Timeout 300
+
+#
+# KeepAlive: Whether or not to allow persistent connections (more than
+# one request per connection). Set to "Off" to deactivate.
+#
+KeepAlive On
+
+# These need to be set in /etc/apache2/envvars
+User ${APACHE_RUN_USER}
+Group ${APACHE_RUN_GROUP}
+
+ErrorLog ${APACHE_LOG_DIR}/error.log
+
+LogLevel warn
+
+# Include module configuration:
+IncludeOptional mods-enabled/*.load
+IncludeOptional mods-enabled/*.conf
+
+<Directory />
+	Options FollowSymLinks
+	AllowOverride None
+	Require all denied
+</Directory>
+
+<Directory /var/www/>
+	Options Indexes FollowSymLinks
+	AllowOverride None
+	Require all granted
+</Directory>
+
+# Include generic snippets of statements
+IncludeOptional conf-enabled/
+
+# Include the virtual host configurations:
+IncludeOptional sites-enabled/*.conf
+
+Define COMPLEX
+
+Define tls_port 1234
+Define tls_port_str "1234"
+
+Define fnmatch_filename test_fnmatch.conf
+
+
+Include test_variables.conf
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/conf-enabled/dummy.conf

@@ -0,0 +1,9 @@
+# 3 - one arg directives
+# 2 - two arg directives
+# 1 - three arg directives
+TestArgsDirective one_arg
+TestArgsDirective one_arg two_arg
+TestArgsDirective one_arg
+TestArgsDirective one_arg two_arg
+TestArgsDirective one_arg two_arg three_arg
+TestArgsDirective one_arg

letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_fnmatch.conf

@@ -0,0 +1 @@
+FNMATCH_DIRECTIVE Success

letsencrypt-apache/letsencrypt_apache/tests/testdata/complex_parsing/test_variables.conf

@@ -0,0 +1,66 @@
+TestVariablePort ${tls_port}
+TestVariablePortStr "${tls_port_str}"
+
+LoadModule status_module modules/mod_status.so
+
+# Basic IfDefine
+<IfDefine COMPLEX>
+    VAR_DIRECTIVE success
+    LoadModule ssl_module modules/mod_ssl.so
+</IfDefine>
+
+<IfDefine !COMPLEX>
+    INVALID_VAR_DIRECTIVE failure
+</IfDefine>
+
+<IfDefine NOT_COMPLEX>
+    INVALID_VAR_DIRECTIVE failure
+</IfDefine>
+
+<IfDefine !NOT_COMPLEX>
+    VAR_DIRECTIVE failure
+</IfDefine>
+
+
+# Basic IfModule
+<IfModule ssl_module>
+    MOD_DIRECTIVE Success
+</IfModule>
+
+<IfModule !ssl_module>
+    INVALID_MOD_DIRECTIVE failure
+</IfModule>
+
+<IfModule fake_module>
+    INVALID_MOD_DIRECTIVE failure
+</IfModule>
+
+<IfModule !fake_module>
+    MOD_DIRECTIVE Success
+</IfModule>
+
+# Nested Tests
+<IfModule status_module>
+    <IfDefine COMPLEX>
+        NESTED_DIRECTIVE success
+
+        <IfModule mod_ssl.c>
+            NESTED_DIRECTIVE success
+        </IfModule>
+
+        <IfModule !mod_ssl.c>
+            INVALID_NESTED_DIRECTIVE failure
+        </IfModule>
+    </IfDefine>
+
+    <IfDefine !COMPLEX>
+        INVALID_NESTED_DIRECTIVE failure
+
+        <IfModule ssl_module>
+            INVALID_NESTED_DIRECTIVE failure
+        </IfModule>
+    </IfDefine>
+
+    NESTED_DIRECTIVE success
+
+</IfModule>

letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/conf-available/bad_conf_file.conf

@@ -0,0 +1,5 @@
+<VirtualHost 1.1.1.1>
+
+ServerName invalid.net
+
+</virtualHost>

letsencrypt-apache/letsencrypt_apache/tests/testdata/debian_apache_2_4/two_vhost_80/apache2/mods-available/rewrite.load

@@ -0,0 +1 @@
+LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so

letsencrypt-apache/letsencrypt_apache/tests/util.py

@@ -1,12 +1,20 @@
 """Common utilities for letsencrypt_apache."""
 import os
-import pkg_resources
+import sys
 import unittest
 
+import augeas
 import mock
+import zope.component
+
+from acme import jose
+
+from letsencrypt.display import util as display_util
 
 from letsencrypt.plugins import common
 
+from letsencrypt.tests import test_util
+
 from letsencrypt_apache import configurator
 from letsencrypt_apache import constants
 from letsencrypt_apache import obj
@@ -14,49 +22,78 @@ from letsencrypt_apache import obj
 
 class ApacheTest(unittest.TestCase):  # pylint: disable=too-few-public-methods
 
-    def setUp(self):
+    def setUp(self, test_dir="debian_apache_2_4/two_vhost_80",
+              config_root="debian_apache_2_4/two_vhost_80/apache2"):
+        # pylint: disable=arguments-differ
         super(ApacheTest, self).setUp()
 
         self.temp_dir, self.config_dir, self.work_dir = common.dir_setup(
-            test_dir="debian_apache_2_4/two_vhost_80",
+            test_dir=test_dir,
             pkg="letsencrypt_apache.tests")
 
         self.ssl_options = common.setup_ssl_options(
             self.config_dir, constants.MOD_SSL_CONF_SRC,
             constants.MOD_SSL_CONF_DEST)
 
-        self.config_path = os.path.join(
-            self.temp_dir, "debian_apache_2_4/two_vhost_80/apache2")
+        self.config_path = os.path.join(self.temp_dir, config_root)
 
-        self.rsa256_file = pkg_resources.resource_filename(
-            "letsencrypt.tests", os.path.join("testdata", "rsa256_key.pem"))
-        self.rsa256_pem = pkg_resources.resource_string(
-            "letsencrypt.tests", os.path.join("testdata", "rsa256_key.pem"))
+        self.rsa512jwk = jose.JWKRSA.load(test_util.load_vector(
+            "rsa512_key.pem"))
+
+
+class ParserTest(ApacheTest):  # pytlint: disable=too-few-public-methods
+
+    def setUp(self, test_dir="debian_apache_2_4/two_vhost_80",
+              config_root="debian_apache_2_4/two_vhost_80/apache2"):
+        super(ParserTest, self).setUp(test_dir, config_root)
+
+        zope.component.provideUtility(display_util.FileDisplay(sys.stdout))
+
+        from letsencrypt_apache.parser import ApacheParser
+        self.aug = augeas.Augeas(
+            flags=augeas.Augeas.NONE | augeas.Augeas.NO_MODL_AUTOLOAD)
+        with mock.patch("letsencrypt_apache.parser.ApacheParser."
+                        "update_runtime_variables"):
+            self.parser = ApacheParser(
+                self.aug, self.config_path, "dummy_ctl_path")
 
 
 def get_apache_configurator(
-        config_path, config_dir, work_dir, version=(2, 4, 7)):
-    """Create an Apache Configurator with the specified options."""
+        config_path, config_dir, work_dir, version=(2, 4, 7), conf=None):
+    """Create an Apache Configurator with the specified options.
 
+    :param conf: Function that returns binary paths. self.conf in Configurator
+
+    """
     backups = os.path.join(work_dir, "backups")
+    mock_le_config = mock.MagicMock(
+        apache_server_root=config_path,
+        apache_le_vhost_ext=constants.CLI_DEFAULTS["le_vhost_ext"],
+        backup_dir=backups,
+        config_dir=config_dir,
+        temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
+        in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
+        work_dir=work_dir)
 
     with mock.patch("letsencrypt_apache.configurator."
                     "subprocess.Popen") as mock_popen:
-        # This just states that the ssl module is already loaded
-        mock_popen().communicate.return_value = ("ssl_module", "")
-        config = configurator.ApacheConfigurator(
-            config=mock.MagicMock(
-                apache_server_root=config_path,
-                apache_le_vhost_ext=constants.CLI_DEFAULTS["le_vhost_ext"],
-                backup_dir=backups,
-                config_dir=config_dir,
-                temp_checkpoint_dir=os.path.join(work_dir, "temp_checkpoints"),
-                in_progress_dir=os.path.join(backups, "IN_PROGRESS"),
-                work_dir=work_dir),
-            name="apache",
-            version=version)
+        # This indicates config_test passes
+        mock_popen().communicate.return_value = ("Fine output", "No problems")
+        mock_popen().returncode = 0
+        with mock.patch("letsencrypt_apache.configurator.le_util."
+                        "exe_exists") as mock_exe_exists:
+            mock_exe_exists.return_value = True
+            with mock.patch("letsencrypt_apache.parser.ApacheParser."
+                            "update_runtime_variables"):
+                config = configurator.ApacheConfigurator(
+                    config=mock_le_config,
+                    name="apache",
+                    version=version)
+                # This allows testing scripts to set it a bit more quickly
+                if conf is not None:
+                    config.conf = conf  # pragma: no cover
 
-    config.prepare()
+                config.prepare()
 
     return config
 
@@ -71,23 +108,23 @@ def get_vh_truth(temp_dir, config_name):
             obj.VirtualHost(
                 os.path.join(prefix, "encryption-example.conf"),
                 os.path.join(aug_pre, "encryption-example.conf/VirtualHost"),
-                set([common.Addr.fromstring("*:80")]),
-                False, True, set(["encryption-example.demo"])),
+                set([obj.Addr.fromstring("*:80")]),
+                False, True, "encryption-example.demo"),
             obj.VirtualHost(
                 os.path.join(prefix, "default-ssl.conf"),
                 os.path.join(aug_pre, "default-ssl.conf/IfModule/VirtualHost"),
-                set([common.Addr.fromstring("_default_:443")]), True, False),
+                set([obj.Addr.fromstring("_default_:443")]), True, False),
             obj.VirtualHost(
                 os.path.join(prefix, "000-default.conf"),
                 os.path.join(aug_pre, "000-default.conf/VirtualHost"),
-                set([common.Addr.fromstring("*:80")]), False, True,
-                set(["ip-172-30-0-17"])),
+                set([obj.Addr.fromstring("*:80")]), False, True,
+                "ip-172-30-0-17"),
             obj.VirtualHost(
                 os.path.join(prefix, "letsencrypt.conf"),
                 os.path.join(aug_pre, "letsencrypt.conf/VirtualHost"),
-                set([common.Addr.fromstring("*:80")]), False, True,
-                set(["letsencrypt.demo"])),
+                set([obj.Addr.fromstring("*:80")]), False, True,
+                "letsencrypt.demo"),
         ]
         return vh_truth
 
-    return None
+    return None  # pragma: no cover

letsencrypt-apache/setup.py

@@ -1,23 +1,56 @@
+import sys
+
 from setuptools import setup
 from setuptools import find_packages
 
 
+version = '0.1.0.dev0'
+
 install_requires = [
-    'acme',
-    'letsencrypt',
-    'mock<1.1.0',  # py26
+    'acme=={0}'.format(version),
+    'letsencrypt=={0}'.format(version),
     'python-augeas',
+    'setuptools',  # pkg_resources
     'zope.component',
     'zope.interface',
 ]
 
+if sys.version_info < (2, 7):
+    install_requires.append('mock<1.1.0')
+else:
+    install_requires.append('mock')
+
 setup(
     name='letsencrypt-apache',
+    version=version,
+    description="Apache plugin for Let's Encrypt client",
+    url='https://github.com/letsencrypt/letsencrypt',
+    author="Let's Encrypt Project",
+    author_email='client-dev@letsencrypt.org',
+    license='Apache License 2.0',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Environment :: Plugins',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Topic :: Internet :: WWW/HTTP',
+        'Topic :: Security',
+        'Topic :: System :: Installation/Setup',
+        'Topic :: System :: Networking',
+        'Topic :: System :: Systems Administration',
+        'Topic :: Utilities',
+    ],
+
     packages=find_packages(),
+    include_package_data=True,
     install_requires=install_requires,
     entry_points={
         'letsencrypt.plugins': [
             'apache = letsencrypt_apache.configurator:ApacheConfigurator',
-         ],
+        ],
     },
 )

letsencrypt-compatibility-test/LICENSE.txt

@@ -0,0 +1,190 @@
+   Copyright 2015 Electronic Frontier Foundation and others
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS

letsencrypt-compatibility-test/MANIFEST.in

@@ -0,0 +1,6 @@
+include LICENSE.txt
+include README.rst
+include letsencrypt_compatibility_test/configurators/apache/a2enmod.sh
+include letsencrypt_compatibility_test/configurators/apache/a2dismod.sh
+include letsencrypt_compatibility_test/configurators/apache/Dockerfile
+recursive-include letsencrypt_compatibility_test/testdata *

letsencrypt-compatibility-test/README.rst

@@ -0,0 +1 @@
+Compatibility tests for Let's Encrypt client

letsencrypt-compatibility-test/letsencrypt_compatibility_test/__init__.py

@@ -0,0 +1 @@
+"""Let's Encrypt compatibility test"""

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/__init__.py

@@ -0,0 +1 @@
+"""Let's Encrypt compatibility test configurators"""

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/Dockerfile

@@ -0,0 +1,20 @@
+FROM httpd
+MAINTAINER Brad Warren <bradmw@umich.edu>
+
+RUN mkdir /var/run/apache2
+
+ENV APACHE_RUN_USER=daemon \
+    APACHE_RUN_GROUP=daemon \
+    APACHE_PID_FILE=/usr/local/apache2/logs/httpd.pid \
+    APACHE_RUN_DIR=/var/run/apache2 \
+    APACHE_LOCK_DIR=/var/lock \
+    APACHE_LOG_DIR=/usr/local/apache2/logs 
+
+COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh /usr/local/bin/
+COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2dismod.sh /usr/local/bin/
+COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/testdata/rsa1024_key2.pem /usr/local/apache2/conf/
+COPY letsencrypt-compatibility-test/letsencrypt_compatibility_test/testdata/empty_cert.pem /usr/local/apache2/conf/
+
+# Note: this only exposes the port to other docker containers. You
+# still have to bind to 443@host at runtime.
+EXPOSE 443

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/__init__.py

@@ -0,0 +1 @@
+"""Let's Encrypt compatibility test Apache configurators"""

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2dismod.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# An extremely simplified version of `a2enmod` for disabling modules in the
+# httpd docker image. First argument is the server_root and the second is the
+# module to be disabled.
+
+apache_confdir=$1
+module=$2
+
+sed -i "/.*"$module".*/d" "$apache_confdir/test.conf"
+enabled_conf="$apache_confdir/mods-enabled/"$module".conf"
+if [ -e "$enabled_conf" ]
+then
+    rm $enabled_conf
+fi

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/a2enmod.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# An extremely simplified version of `a2enmod` for enabling modules in the
+# httpd docker image. First argument is the Apache ServerRoot which should be
+# an absolute path. The second is the module to be enabled, such as `ssl`.
+
+confdir=$1
+module=$2
+
+echo "LoadModule ${module}_module " \
+    "/usr/local/apache2/modules/mod_${module}.so" >> "${confdir}/test.conf"
+availbase="/mods-available/${module}.conf"
+availconf=$confdir$availbase
+enabldir="$confdir/mods-enabled"
+enablconf="$enabldir/${module}.conf"
+if [ -e $availconf -a -d $enabldir -a ! -e $enablconf ]
+then
+    ln -s "..$availbase" $enablconf
+fi

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/apache24.py

@@ -0,0 +1,64 @@
+"""Proxies ApacheConfigurator for Apache 2.4 tests"""
+
+import zope.interface
+
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import interfaces
+from letsencrypt_compatibility_test.configurators.apache import common as apache_common
+
+
+# The docker image doesn't actually have the watchdog module, but unless the
+# config uses mod_heartbeat or mod_heartmonitor (which aren't installed and
+# therefore the config won't be loaded), I believe this isn't a problem
+# http://httpd.apache.org/docs/2.4/mod/mod_watchdog.html
+STATIC_MODULES = set(["core", "so", "http", "mpm_event", "watchdog"])
+
+
+SHARED_MODULES = {
+    "log_config", "logio", "version", "unixd", "access_compat", "actions",
+    "alias", "allowmethods", "auth_basic", "auth_digest", "auth_form",
+    "authn_anon", "authn_core", "authn_dbd", "authn_dbm", "authn_file",
+    "authn_socache", "authnz_ldap", "authz_core", "authz_dbd", "authz_dbm",
+    "authz_groupfile", "authz_host", "authz_owner", "authz_user", "autoindex",
+    "buffer", "cache", "cache_disk", "cache_socache", "cgid", "dav", "dav_fs",
+    "dbd", "deflate", "dir", "dumpio", "env", "expires", "ext_filter",
+    "file_cache", "filter", "headers", "include", "info", "lbmethod_bybusyness",
+    "lbmethod_byrequests", "lbmethod_bytraffic", "lbmethod_heartbeat", "ldap",
+    "log_debug", "macro", "mime", "negotiation", "proxy", "proxy_ajp",
+    "proxy_balancer", "proxy_connect", "proxy_express", "proxy_fcgi",
+    "proxy_ftp", "proxy_http", "proxy_scgi", "proxy_wstunnel", "ratelimit",
+    "remoteip", "reqtimeout", "request", "rewrite", "sed", "session",
+    "session_cookie", "session_crypto", "session_dbd", "setenvif",
+    "slotmem_shm", "socache_dbm", "socache_memcache", "socache_shmcb",
+    "speling", "ssl", "status", "substitute", "unique_id", "userdir",
+    "vhost_alias"}
+
+
+class Proxy(apache_common.Proxy):
+    """Wraps the ApacheConfigurator for Apache 2.4 tests"""
+
+    zope.interface.implements(interfaces.IConfiguratorProxy)
+
+    def __init__(self, args):
+        """Initializes the plugin with the given command line args"""
+        super(Proxy, self).__init__(args)
+        # Running init isn't ideal, but the Docker container needs to survive
+        # Apache restarts
+        self.start_docker("bradmw/apache2.4", "init")
+
+    def preprocess_config(self, server_root):
+        """Prepares the configuration for use in the Docker"""
+        super(Proxy, self).preprocess_config(server_root)
+        if self.version[1] != 4:
+            raise errors.Error("Apache version not 2.4")
+
+        with open(self.test_conf, "a") as f:
+            for module in self.modules:
+                if module not in STATIC_MODULES:
+                    if module in SHARED_MODULES:
+                        f.write(
+                            "LoadModule {0}_module /usr/local/apache2/modules/"
+                            "mod_{0}.so\n".format(module))
+                    else:
+                        raise errors.Error(
+                            "Unsupported module {0}".format(module))

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/apache/common.py

@@ -0,0 +1,285 @@
+"""Provides a common base for Apache proxies"""
+import re
+import os
+import subprocess
+
+import mock
+import zope.interface
+
+from letsencrypt import configuration
+from letsencrypt import errors as le_errors
+from letsencrypt_apache import configurator
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import interfaces
+from letsencrypt_compatibility_test import util
+from letsencrypt_compatibility_test.configurators import common as configurators_common
+
+
+APACHE_VERSION_REGEX = re.compile(r"Apache/([0-9\.]*)", re.IGNORECASE)
+APACHE_COMMANDS = ["apachectl", "a2enmod", "a2dismod"]
+
+
+class Proxy(configurators_common.Proxy):
+    # pylint: disable=too-many-instance-attributes
+    """A common base for Apache test configurators"""
+
+    zope.interface.implements(interfaces.IConfiguratorProxy)
+
+    def __init__(self, args):
+        """Initializes the plugin with the given command line args"""
+        super(Proxy, self).__init__(args)
+        self.le_config.apache_le_vhost_ext = "-le-ssl.conf"
+
+        self._setup_mock()
+
+        self.modules = self.server_root = self.test_conf = self.version = None
+        self._apache_configurator = self._all_names = self._test_names = None
+
+    def _setup_mock(self):
+        """Replaces specific modules with mock.MagicMock"""
+        mock_subprocess = mock.MagicMock()
+        mock_subprocess.check_call = self.check_call
+        mock_subprocess.Popen = self.popen
+
+        mock.patch(
+            "letsencrypt_apache.configurator.subprocess",
+            mock_subprocess).start()
+        mock.patch(
+            "letsencrypt_apache.parser.subprocess",
+            mock_subprocess).start()
+        mock.patch(
+            "letsencrypt.le_util.subprocess",
+            mock_subprocess).start()
+        mock.patch(
+            "letsencrypt_apache.configurator.le_util.exe_exists",
+            _is_apache_command).start()
+
+        patch = mock.patch(
+            "letsencrypt_apache.configurator.display_ops.select_vhost")
+        mock_display = patch.start()
+        mock_display.side_effect = le_errors.PluginError(
+            "Unable to determine vhost")
+
+    def check_call(self, command, *args, **kwargs):
+        """If command is an Apache command, command is executed in the
+        running docker image. Otherwise, subprocess.check_call is used.
+
+        """
+        if _is_apache_command(command):
+            command = _modify_command(command)
+            return super(Proxy, self).check_call(command, *args, **kwargs)
+        else:
+            return subprocess.check_call(command, *args, **kwargs)
+
+    def popen(self, command, *args, **kwargs):
+        """If command is an Apache command, command is executed in the
+        running docker image. Otherwise, subprocess.Popen is used.
+
+        """
+        if _is_apache_command(command):
+            command = _modify_command(command)
+            return super(Proxy, self).popen(command, *args, **kwargs)
+        else:
+            return subprocess.Popen(command, *args, **kwargs)
+
+    def __getattr__(self, name):
+        """Wraps the Apache Configurator methods"""
+        method = getattr(self._apache_configurator, name, None)
+        if callable(method):
+            return method
+        else:
+            raise AttributeError()
+
+    def load_config(self):
+        """Loads the next configuration for the plugin to test"""
+        if hasattr(self.le_config, "apache_init_script"):
+            try:
+                self.check_call([self.le_config.apache_init_script, "stop"])
+            except errors.Error:
+                raise errors.Error(
+                    "Failed to stop previous apache config from running")
+
+        config = super(Proxy, self).load_config()
+        self.modules = _get_modules(config)
+        self.version = _get_version(config)
+        self._all_names, self._test_names = _get_names(config)
+
+        server_root = _get_server_root(config)
+        with open(os.path.join(config, "config_file")) as f:
+            config_file = os.path.join(server_root, f.readline().rstrip())
+        self.test_conf = _create_test_conf(server_root, config_file)
+
+        self.preprocess_config(server_root)
+        self._prepare_configurator(server_root, config_file)
+
+        try:
+            self.check_call("apachectl -d {0} -f {1} -k start".format(
+                server_root, config_file))
+        except errors.Error:
+            raise errors.Error(
+                "Apache failed to load {0} before tests started".format(
+                    config))
+
+        return config
+
+    def preprocess_config(self, server_root):
+        # pylint: disable=anomalous-backslash-in-string, no-self-use
+        """Prepares the configuration for use in the Docker"""
+
+        find = subprocess.Popen(
+            ["find", server_root, "-type", "f"],
+            stdout=subprocess.PIPE)
+        subprocess.check_call([
+            "xargs", "sed", "-e", "s/DocumentRoot.*/DocumentRoot "
+            "\/usr\/local\/apache2\/htdocs/I",
+            "-e", "s/SSLPassPhraseDialog.*/SSLPassPhraseDialog builtin/I",
+            "-e", "s/TypesConfig.*/TypesConfig "
+            "\/usr\/local\/apache2\/conf\/mime.types/I",
+            "-e", "s/LoadModule/#LoadModule/I",
+            "-e", "s/SSLCertificateFile.*/SSLCertificateFile "
+            "\/usr\/local\/apache2\/conf\/empty_cert.pem/I",
+            "-e", "s/SSLCertificateKeyFile.*/SSLCertificateKeyFile "
+            "\/usr\/local\/apache2\/conf\/rsa1024_key2.pem/I",
+            "-i"], stdin=find.stdout)
+
+    def _prepare_configurator(self, server_root, config_file):
+        """Prepares the Apache plugin for testing"""
+        self.le_config.apache_server_root = server_root
+        self.le_config.apache_ctl = "apachectl -d {0} -f {1}".format(
+            server_root, config_file)
+        self.le_config.apache_enmod = "a2enmod.sh {0}".format(server_root)
+        self.le_config.apache_dismod = "a2dismod.sh {0}".format(server_root)
+        self.le_config.apache_init_script = self.le_config.apache_ctl + " -k"
+
+        self._apache_configurator = configurator.ApacheConfigurator(
+            config=configuration.NamespaceConfig(self.le_config),
+            name="apache")
+        self._apache_configurator.prepare()
+
+    def cleanup_from_tests(self):
+        """Performs any necessary cleanup from running plugin tests"""
+        super(Proxy, self).cleanup_from_tests()
+        mock.patch.stopall()
+
+    def get_all_names_answer(self):
+        """Returns the set of domain names that the plugin should find"""
+        if self._all_names:
+            return self._all_names
+        else:
+            raise errors.Error("No configuration file loaded")
+
+    def get_testable_domain_names(self):
+        """Returns the set of domain names that can be tested against"""
+        if self._test_names:
+            return self._test_names
+        else:
+            return {"example.com"}
+
+    def deploy_cert(self, domain, cert_path, key_path, chain_path=None,
+            fullchain_path=None):
+        """Installs cert"""
+        cert_path, key_path, chain_path = self.copy_certs_and_keys(
+            cert_path, key_path, chain_path)
+        self._apache_configurator.deploy_cert(
+            domain, cert_path, key_path, chain_path, fullchain_path)
+
+
+def _is_apache_command(command):
+    """Returns true if command is an Apache command"""
+    if isinstance(command, list):
+        command = command[0]
+
+    for apache_command in APACHE_COMMANDS:
+        if command.startswith(apache_command):
+            return True
+
+    return False
+
+
+def _modify_command(command):
+    """Modifies command so configtest works inside the docker image"""
+    if isinstance(command, list):
+        for i in xrange(len(command)):
+            if command[i] == "configtest":
+                command[i] = "-t"
+    else:
+        command = command.replace("configtest", "-t")
+
+    return command
+
+
+def _create_test_conf(server_root, apache_config):
+    """Creates a test config file and adds it to the Apache config"""
+    test_conf = os.path.join(server_root, "test.conf")
+    open(test_conf, "w").close()
+    subprocess.check_call(
+        ["sed", "-i", "1iInclude test.conf", apache_config])
+    return test_conf
+
+
+def _get_server_root(config):
+    """Returns the server root directory in config"""
+    subdirs = [
+        name for name in os.listdir(config)
+        if os.path.isdir(os.path.join(config, name))]
+
+    if len(subdirs) != 1:
+        errors.Error("Malformed configuration directiory {0}".format(config))
+
+    return os.path.join(config, subdirs[0].rstrip())
+
+
+def _get_names(config):
+    """Returns all and testable domain names in config"""
+    all_names = set()
+    non_ip_names = set()
+    with open(os.path.join(config, "vhosts")) as f:
+        for line in f:
+            # If parsing a specific vhost
+            if line[0].isspace():
+                words = line.split()
+                if words[0] == "alias":
+                    all_names.add(words[1])
+                    non_ip_names.add(words[1])
+                # If for port 80 and not IP vhost
+                elif words[1] == "80" and not util.IP_REGEX.match(words[3]):
+                    all_names.add(words[3])
+                    non_ip_names.add(words[3])
+            elif "NameVirtualHost" not in line:
+                words = line.split()
+                if (words[0].endswith("*") or words[0].endswith("80") and
+                        not util.IP_REGEX.match(words[1]) and
+                        words[1].find(".") != -1):
+                    all_names.add(words[1])
+    return all_names, non_ip_names
+
+
+def _get_modules(config):
+    """Returns the list of modules found in module_list"""
+    modules = []
+    with open(os.path.join(config, "modules")) as f:
+        for line in f:
+            # Modules list is indented, everything else is headers/footers
+            if line[0].isspace():
+                words = line.split()
+                # Modules redundantly end in "_module" which we can discard
+                modules.append(words[0][:-7])
+
+    return modules
+
+
+def _get_version(config):
+    """Return version of Apache Server.
+
+    Version is returned as tuple. (ie. 2.4.7 = (2, 4, 7)). Code taken from
+    the Apache plugin.
+
+    """
+    with open(os.path.join(config, "version")) as f:
+        # Should be on first line of input
+        matches = APACHE_VERSION_REGEX.findall(f.readline())
+
+    if len(matches) != 1:
+        raise errors.Error("Unable to find Apache version")
+
+    return tuple([int(i) for i in matches[0].split(".")])

letsencrypt-compatibility-test/letsencrypt_compatibility_test/configurators/common.py

@@ -0,0 +1,145 @@
+"""Provides a common base for configurator proxies"""
+import logging
+import os
+import shutil
+import tempfile
+
+import docker
+
+from letsencrypt import constants
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import util
+
+
+logger = logging.getLogger(__name__)
+
+
+class Proxy(object):
+    # pylint: disable=too-many-instance-attributes
+    """A common base for compatibility test configurators"""
+
+    _NOT_ADDED_ARGS = True
+
+    @classmethod
+    def add_parser_arguments(cls, parser):
+        """Adds command line arguments needed by the plugin"""
+        if Proxy._NOT_ADDED_ARGS:
+            group = parser.add_argument_group("docker")
+            group.add_argument(
+                "--docker-url", default="unix://var/run/docker.sock",
+                help="URL of the docker server")
+            group.add_argument(
+                "--no-remove", action="store_true",
+                help="do not delete container on program exit")
+            Proxy._NOT_ADDED_ARGS = False
+
+    def __init__(self, args):
+        """Initializes the plugin with the given command line args"""
+        self._temp_dir = tempfile.mkdtemp()
+        self.le_config = util.create_le_config(self._temp_dir)
+        config_dir = util.extract_configs(args.configs, self._temp_dir)
+        self._configs = [
+            os.path.join(config_dir, config)
+            for config in os.listdir(config_dir)]
+
+        self.args = args
+        self._docker_client = docker.Client(
+            base_url=self.args.docker_url, version="auto")
+        self.http_port, self.https_port = util.get_two_free_ports()
+        self._container_id = None
+
+    def has_more_configs(self):
+        """Returns true if there are more configs to test"""
+        return bool(self._configs)
+
+    def cleanup_from_tests(self):
+        """Performs any necessary cleanup from running plugin tests"""
+        self._docker_client.stop(self._container_id, 0)
+        if not self.args.no_remove:
+            self._docker_client.remove_container(self._container_id)
+
+    def load_config(self):
+        """Returns the next config directory to be tested"""
+        shutil.rmtree(self.le_config.work_dir, ignore_errors=True)
+        backup = os.path.join(self.le_config.work_dir, constants.BACKUP_DIR)
+        os.makedirs(backup)
+        return self._configs.pop()
+
+    def start_docker(self, image_name, command):
+        """Creates and runs a Docker container with the specified image"""
+        logger.warning("Pulling Docker image. This may take a minute.")
+        for line in self._docker_client.pull(image_name, stream=True):
+            logger.debug(line)
+
+        host_config = docker.utils.create_host_config(
+            binds={self._temp_dir: {"bind": self._temp_dir, "mode": "rw"}},
+            port_bindings={
+                80: ("127.0.0.1", self.http_port),
+                443: ("127.0.0.1", self.https_port)},)
+        container = self._docker_client.create_container(
+            image_name, command, ports=[80, 443], volumes=self._temp_dir,
+            host_config=host_config)
+        if container["Warnings"]:
+            logger.warning(container["Warnings"])
+        self._container_id = container["Id"]
+        self._docker_client.start(self._container_id)
+
+    def check_call(self, command, *args, **kwargs):
+        # pylint: disable=unused-argument
+        """Simulates a call to check_call but executes the command in the
+        running docker image
+
+        """
+        if self.popen(command).returncode:
+            raise errors.Error(
+                "{0} exited with a nonzero value".format(command))
+
+    def popen(self, command, *args, **kwargs):
+        # pylint: disable=unused-argument
+        """Simulates a call to Popen but executes the command in the
+        running docker image
+
+        """
+        class SimplePopen(object):
+            # pylint: disable=too-few-public-methods
+            """Simplified Popen object"""
+            def __init__(self, returncode, output):
+                self.returncode = returncode
+                self._stdout = output
+                self._stderr = output
+
+            def communicate(self):
+                """Returns stdout and stderr"""
+                return self._stdout, self._stderr
+
+        if isinstance(command, list):
+            command = " ".join(command)
+
+        returncode, output = self.execute_in_docker(command)
+        return SimplePopen(returncode, output)
+
+    def execute_in_docker(self, command):
+        """Executes command inside the running docker image"""
+        logger.debug("Executing '%s'", command)
+        exec_id = self._docker_client.exec_create(self._container_id, command)
+        output = self._docker_client.exec_start(exec_id)
+        returncode = self._docker_client.exec_inspect(exec_id)["ExitCode"]
+        return returncode, output
+
+    def copy_certs_and_keys(self, cert_path, key_path, chain_path=None):
+        """Copies certs and keys into the temporary directory"""
+        cert_and_key_dir = os.path.join(self._temp_dir, "certs_and_keys")
+        if not os.path.isdir(cert_and_key_dir):
+            os.mkdir(cert_and_key_dir)
+
+        cert = os.path.join(cert_and_key_dir, "cert")
+        shutil.copy(cert_path, cert)
+        key = os.path.join(cert_and_key_dir, "key")
+        shutil.copy(key_path, key)
+        if chain_path:
+            chain = os.path.join(cert_and_key_dir, "chain")
+            shutil.copy(chain_path, chain)
+        else:
+            chain = None
+
+        return cert, key, chain

letsencrypt-compatibility-test/letsencrypt_compatibility_test/errors.py

@@ -0,0 +1,5 @@
+"""Let's Encrypt compatibility test errors"""
+
+
+class Error(Exception):
+    """Generic Let's Encrypt compatibility test error"""

letsencrypt-compatibility-test/letsencrypt_compatibility_test/interfaces.py

@@ -0,0 +1,52 @@
+"""Let's Encrypt compatibility test interfaces"""
+import zope.interface
+
+import letsencrypt.interfaces
+
+# pylint: disable=no-self-argument,no-method-argument
+
+
+class IPluginProxy(zope.interface.Interface):
+    """Wraps a Let's Encrypt plugin"""
+    http_port = zope.interface.Attribute(
+        "The port to connect to on localhost for HTTP traffic")
+
+    https_port = zope.interface.Attribute(
+        "The port to connect to on localhost for HTTPS traffic")
+
+    def add_parser_arguments(cls, parser):
+        """Adds command line arguments needed by the parser"""
+
+    def __init__(args):
+        """Initializes the plugin with the given command line args"""
+
+    def cleanup_from_tests():
+        """Performs any necessary cleanup from running plugin tests.
+
+        This is guaranteed to be called before the program exits.
+
+        """
+
+    def has_more_configs():
+        """Returns True if there are more configs to test"""
+
+    def load_config():
+        """Loads the next config and returns its name"""
+
+    def get_testable_domain_names():
+        """Returns the domain names that can be used in testing"""
+
+
+class IAuthenticatorProxy(IPluginProxy, letsencrypt.interfaces.IAuthenticator):
+    """Wraps a Let's Encrypt authenticator"""
+
+
+class IInstallerProxy(IPluginProxy, letsencrypt.interfaces.IInstaller):
+    """Wraps a Let's Encrypt installer"""
+
+    def get_all_names_answer():
+        """Returns all names that should be found by the installer"""
+
+
+class IConfiguratorProxy(IAuthenticatorProxy, IInstallerProxy):
+    """Wraps a Let's Encrypt configurator"""

letsencrypt-compatibility-test/letsencrypt_compatibility_test/test_driver.py

@@ -0,0 +1,368 @@
+"""Tests Let's Encrypt plugins against different server configurations."""
+import argparse
+import filecmp
+import functools
+import logging
+import os
+import shutil
+import tempfile
+import time
+
+import OpenSSL
+
+from acme import challenges
+from acme import crypto_util
+from acme import messages
+from letsencrypt import achallenges
+from letsencrypt import errors as le_errors
+from letsencrypt import validator
+from letsencrypt.tests import acme_util
+
+from letsencrypt_compatibility_test import errors
+from letsencrypt_compatibility_test import util
+from letsencrypt_compatibility_test.configurators.apache import apache24
+
+
+DESCRIPTION = """
+Tests Let's Encrypt plugins against different server configuratons. It is
+assumed that Docker is already installed. If no test types is specified, all
+tests that the plugin supports are performed.
+
+"""
+
+PLUGINS = {"apache": apache24.Proxy}
+
+
+logger = logging.getLogger(__name__)
+
+
+def test_authenticator(plugin, config, temp_dir):
+    """Tests authenticator, returning True if the tests are successful"""
+    backup = _create_backup(config, temp_dir)
+
+    achalls = _create_achalls(plugin)
+    if not achalls:
+        logger.error("The plugin and this program support no common "
+                     "challenge types")
+        return False
+
+    try:
+        responses = plugin.perform(achalls)
+    except le_errors.Error as error:
+        logger.error("Performing challenges on %s caused an error:", config)
+        logger.exception(error)
+        return False
+
+    success = True
+    for i in xrange(len(responses)):
+        if not responses[i]:
+            logger.error(
+                "Plugin failed to complete %s for %s in %s",
+                type(achalls[i]), achalls[i].domain, config)
+            success = False
+        elif isinstance(responses[i], challenges.DVSNIResponse):
+            verify = functools.partial(responses[i].simple_verify, achalls[i],
+                                       achalls[i].domain,
+                                       util.JWK.public_key(),
+                                       host="127.0.0.1",
+                                       port=plugin.https_port)
+            if _try_until_true(verify):
+                logger.info(
+                    "DVSNI verification for %s succeeded", achalls[i].domain)
+            else:
+                logger.error(
+                    "DVSNI verification for %s in %s failed",
+                    achalls[i].domain, config)
+                success = False
+
+    if success:
+        try:
+            plugin.cleanup(achalls)
+        except le_errors.Error as error:
+            logger.error("Challenge cleanup for %s caused an error:", config)
+            logger.exception(error)
+            success = False
+
+        if _dirs_are_unequal(config, backup):
+            logger.error("Challenge cleanup failed for %s", config)
+            return False
+        else:
+            logger.info("Challenge cleanup succeeded")
+
+    return success
+
+
+def _create_achalls(plugin):
+    """Returns a list of annotated challenges to test on plugin"""
+    achalls = list()
+    names = plugin.get_testable_domain_names()
+    for domain in names:
+        prefs = plugin.get_chall_pref(domain)
+        for chall_type in prefs:
+            if chall_type == challenges.DVSNI:
+                chall = challenges.DVSNI(
+                    token=os.urandom(challenges.DVSNI.TOKEN_SIZE))
+                challb = acme_util.chall_to_challb(
+                    chall, messages.STATUS_PENDING)
+                achall = achallenges.DVSNI(
+                    challb=challb, domain=domain, account_key=util.JWK)
+                achalls.append(achall)
+
+    return achalls
+
+
+def test_installer(args, plugin, config, temp_dir):
+    """Tests plugin as an installer"""
+    backup = _create_backup(config, temp_dir)
+
+    names_match = plugin.get_all_names() == plugin.get_all_names_answer()
+    if names_match:
+        logger.info("get_all_names test succeeded")
+    else:
+        logger.error("get_all_names test failed for config %s", config)
+
+    domains = list(plugin.get_testable_domain_names())
+    success = test_deploy_cert(plugin, temp_dir, domains)
+
+    if success and args.enhance:
+        success = test_enhancements(plugin, domains)
+
+    good_rollback = test_rollback(plugin, config, backup)
+    return names_match and success and good_rollback
+
+
+def test_deploy_cert(plugin, temp_dir, domains):
+    """Tests deploy_cert returning True if the tests are successful"""
+    cert = crypto_util.gen_ss_cert(util.KEY, domains)
+    cert_path = os.path.join(temp_dir, "cert.pem")
+    with open(cert_path, "w") as f:
+        f.write(OpenSSL.crypto.dump_certificate(
+            OpenSSL.crypto.FILETYPE_PEM, cert))
+
+    for domain in domains:
+        try:
+            plugin.deploy_cert(domain, cert_path, util.KEY_PATH)
+        except le_errors.Error as error:
+            logger.error("Plugin failed to deploy ceritificate for %s:", domain)
+            logger.exception(error)
+            return False
+
+    if not _save_and_restart(plugin, "deployed"):
+        return False
+
+    success = True
+    for domain in domains:
+        verify = functools.partial(validator.Validator().certificate, cert,
+                                   domain, "127.0.0.1", plugin.https_port)
+        if not _try_until_true(verify):
+            logger.error("Could not verify certificate for domain %s", domain)
+            success = False
+
+    if success:
+        logger.info("HTTPS validation succeeded")
+
+    return success
+
+
+def test_enhancements(plugin, domains):
+    """Tests supported enhancements returning True if successful"""
+    supported = plugin.supported_enhancements()
+
+    if "redirect" not in supported:
+        logger.error("The plugin and this program support no common "
+                     "enhancements")
+        return False
+
+    for domain in domains:
+        try:
+            plugin.enhance(domain, "redirect")
+        except le_errors.PluginError as error:
+            # Don't immediately fail because a redirect may already be enabled
+            logger.warning("Plugin failed to enable redirect for %s:", domain)
+            logger.warning("%s", error)
+        except le_errors.Error as error:
+            logger.error("An error occurred while enabling redirect for %s:",
+                         domain)
+            logger.exception(error)
+
+    if not _save_and_restart(plugin, "enhanced"):
+        return False
+
+    success = True
+    for domain in domains:
+        verify = functools.partial(validator.Validator().redirect, "localhost",
+                                   plugin.http_port, headers={"Host": domain})
+        if not _try_until_true(verify):
+            logger.error("Improper redirect for domain %s", domain)
+            success = False
+
+    if success:
+        logger.info("Enhancments test succeeded")
+
+    return success
+
+
+def _try_until_true(func, max_tries=5, sleep_time=0.5):
+    """Calls func up to max_tries times until it returns True"""
+    for _ in xrange(0, max_tries):
+        if func():
+            return True
+        else:
+            time.sleep(sleep_time)
+
+    return False
+
+
+def _save_and_restart(plugin, title=None):
+    """Saves and restart the plugin, returning True if no errors occurred"""
+    try:
+        plugin.save(title)
+        plugin.restart()
+        return True
+    except le_errors.Error as error:
+        logger.error("Plugin failed to save and restart server:")
+        logger.exception(error)
+        return False
+
+
+def test_rollback(plugin, config, backup):
+    """Tests the rollback checkpoints function"""
+    try:
+        plugin.rollback_checkpoints(1337)
+    except le_errors.Error as error:
+        logger.error("Plugin raised an exception during rollback:")
+        logger.exception(error)
+        return False
+
+    if _dirs_are_unequal(config, backup):
+        logger.error("Rollback failed for config `%s`", config)
+        return False
+    else:
+        logger.info("Rollback succeeded")
+        return True
+
+
+def _create_backup(config, temp_dir):
+    """Creates a backup of config in temp_dir"""
+    backup = os.path.join(temp_dir, "backup")
+    shutil.rmtree(backup, ignore_errors=True)
+    shutil.copytree(config, backup, symlinks=True)
+
+    return backup
+
+
+def _dirs_are_unequal(dir1, dir2):
+    """Returns True if dir1 and dir2 are unequal"""
+    dircmps = [filecmp.dircmp(dir1, dir2)]
+    while len(dircmps):
+        dircmp = dircmps.pop()
+        if dircmp.left_only or dircmp.right_only:
+            logger.error("The following files and directories are only "
+                         "present in one directory")
+            if dircmp.left_only:
+                logger.error(dircmp.left_only)
+            else:
+                logger.error(dircmp.right_only)
+            return True
+        elif dircmp.common_funny or dircmp.funny_files:
+            logger.error("The following files and directories could not be "
+                         "compared:")
+            if dircmp.common_funny:
+                logger.error(dircmp.common_funny)
+            else:
+                logger.error(dircmp.funny_files)
+            return True
+        elif dircmp.diff_files:
+            logger.error("The following files differ:")
+            logger.error(dircmp.diff_files)
+            return True
+
+        for subdir in dircmp.subdirs.itervalues():
+            dircmps.append(subdir)
+
+    return False
+
+
+def get_args():
+    """Returns parsed command line arguments."""
+    parser = argparse.ArgumentParser(
+        description=DESCRIPTION,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    group = parser.add_argument_group("general")
+    group.add_argument(
+        "-c", "--configs", default="configs.tar.gz",
+        help="a directory or tarball containing server configurations")
+    group.add_argument(
+        "-p", "--plugin", default="apache", help="the plugin to be tested")
+    group.add_argument(
+        "-v", "--verbose", dest="verbose_count", action="count",
+        default=0, help="you know how to use this")
+    group.add_argument(
+        "-a", "--auth", action="store_true",
+        help="tests the challenges the plugin supports")
+    group.add_argument(
+        "-i", "--install", action="store_true",
+        help="tests the plugin as an installer")
+    group.add_argument(
+        "-e", "--enhance", action="store_true", help="tests the enhancements "
+        "the plugin supports (implicitly includes installer tests)")
+
+    for plugin in PLUGINS.itervalues():
+        plugin.add_parser_arguments(parser)
+
+    args = parser.parse_args()
+    if args.enhance:
+        args.install = True
+    elif not (args.auth or args.install):
+        args.auth = args.install = args.enhance = True
+
+    return args
+
+
+def setup_logging(args):
+    """Prepares logging for the program"""
+    handler = logging.StreamHandler()
+
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.ERROR - args.verbose_count * 10)
+    root_logger.addHandler(handler)
+
+
+def main():
+    """Main test script execution."""
+    args = get_args()
+    setup_logging(args)
+
+    if args.plugin not in PLUGINS:
+        raise errors.Error("Unknown plugin {0}".format(args.plugin))
+
+    temp_dir = tempfile.mkdtemp()
+    plugin = PLUGINS[args.plugin](args)
+    try:
+        plugin.execute_in_docker("mkdir -p /var/log/apache2")
+        while plugin.has_more_configs():
+            success = True
+
+            try:
+                config = plugin.load_config()
+                logger.info("Loaded configuration: %s", config)
+                if args.auth:
+                    success = test_authenticator(plugin, config, temp_dir)
+                if success and args.install:
+                    success = test_installer(args, plugin, config, temp_dir)
+            except errors.Error as error:
+                logger.error("Tests on %s raised:", config)
+                logger.exception(error)
+                success = False
+
+            if success:
+                logger.info("All tests on %s succeeded", config)
+            else:
+                logger.error("Tests on %s failed", config)
+    finally:
+        plugin.cleanup_from_tests()
+
+
+if __name__ == "__main__":
+    main()

letsencrypt-compatibility-test/letsencrypt_compatibility_test/testdata/empty_cert.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICATCCAWoCCQCvMbKu4FHZ6zANBgkqhkiG9w0BAQsFADBFMQswCQYDVQQGEwJB
+VTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0
+cyBQdHkgTHRkMB4XDTE1MDcyMzIzMjc1MFoXDTE2MDcyMjIzMjc1MFowRTELMAkG
+A1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0
+IFdpZGdpdHMgUHR5IEx0ZDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAws3o
+y46PMLM9Gr68pbex0MhdPr7Cq4rRe9BBpnOuHFdF35Ak0aPrzFwVzLlGOir94U11
+e5JYJDWJi+4FwLBRkOAfanjJ5GJ9BnEHSOdbtO+sv9uhbt+7iYOOUOngKSiJyUrM
+i1THAE+B1CenxZ1KHRQCke708zkK8jVuxLeIAOMCAwEAATANBgkqhkiG9w0BAQsF
+AAOBgQCC3LUP3MHk+IBmwHHZAZCX+6p4lop9SP6y6rDpWgnqEEeb9oFleHi2Rvzq
+7gxl6nS5AsaSzfAygJ3zWKTwVAZyU4GOQ8QTK+nHk3+LO1X4cDbUlQfm5+YuwKDa
+4LFKeovmrK6BiMLIc1J+MxUjLfCeVHYSdkZULTVXue0zif0BUA==
+-----END CERTIFICATE-----