Initial version

This commit is contained in:
Dan Halbert 2023-09-06 00:30:43 -04:00
parent c106cf17c9
commit 5c85c604a0
8 changed files with 2826 additions and 2 deletions

View file

@ -1,2 +1,26 @@
# certificates
TLS/SSL certificates used in Adafruit software
## TLS/SSL certificates used in Adafruit software
[CircuitPython](https://github.com/adafruit/circuitpython),
[NINA-FW](https://github.com/adafruit/nina-fw),
Adafruit IO Arduino libraries, and other Adafruit software need a current set of TLS
root certificates for secure web access.
Microsoft, Mozilla, Android, curl, and other projects maintain lists of root and related certificates.
Those lists are quite complete, and too large for some embedded firmware.
This repo includes tools to download a list of root certificates and
subset it to the most commonly needed roots. Projects can then use
this repo as a submodule to have access to an updated list of root
certificates.
Currently the certificates are filtered from the [`curl` root
list](https://curl.se/docs/caextract.html), which is based on the
Mozilla root list.
- `tools/filter_certs.py` does the filtering to the most common root cert providers.
- `tools/issuers.txt` contains regexps to match those providers.
- `tools/test_site_coverage.py` tests a given `roots.pem` against a long list of URL's.
- `tools/urls.txt` is that list of URLs. Add to it as necessary. Some are commented out, for reasons noted.
The resulting files are in `data/`.
- `data/roots.pem` is just a certificate bundle.
- `data/roots-commented.pem` is the same bundle with a comment line describing the certificate.

1099
data/roots-commented.pem Normal file

File diff suppressed because it is too large Load diff

1053
data/roots.pem Normal file

File diff suppressed because it is too large Load diff

2
requirements.txt Normal file
View file

@ -0,0 +1,2 @@
cryptography>=39.0.0
requests

92
tools/filter_certs.py Executable file
View file

@ -0,0 +1,92 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2023 Dan Halbert for Adafruit Industries LLC
#
# SPDX-License-Identifier: MIT
import click
from cryptography.hazmat.primitives.serialization import Encoding
from cryptography.x509.oid import NameOID
import cryptography.x509
import requests
import re
@click.command()
@click.option(
"--in",
"in_",
help=".pem filename or URL to filter",
default="https://curl.se/ca/cacert.pem",
show_default=True,
)
@click.option(
"--out",
"out_",
default="roots.pem",
help="filtered .pem file",
type=click.File("wb"),
show_default=True,
)
@click.option(
"--comment",
is_flag=True,
default=False,
help="Comment certs in output",
show_default=True,
)
@click.option(
"--issuers",
"issuers_",
help="file of issuers to select; one regexp per line; substring match; case-insensitive; # comments OK",
default="issuers.txt",
type=click.File("r"),
show_default=True,
)
def run(in_, out_, comment, issuers_):
if in_.startswith("http"):
input_text = requests.get(in_).content
else:
with open(in_, "rb") as input:
input_text = input.read()
# Read a list of regexps to substr-match against Issue names.
issuer_patterns = []
for line in issuers_.readlines():
line = line.strip()
if line.startswith("#"):
continue
issuer_patterns.append(re.compile(line, flags=re.IGNORECASE))
# Read in all the certs at once.
input_certs = cryptography.x509.load_pem_x509_certificates(input_text)
# For each cert, see if its O or CN name matches against the list of filter patterns.
for cert in input_certs:
input_issuer = cert.issuer
org_name_attributes = input_issuer.get_attributes_for_oid(
NameOID.ORGANIZATION_NAME
)
org_name = org_name_attributes[0].value if org_name_attributes else ""
common_name_attributes = input_issuer.get_attributes_for_oid(
NameOID.COMMON_NAME
)
common_name = common_name_attributes[0].value if common_name_attributes else ""
match_name = org_name or common_name
if not match_name:
raise ValueError(f"no OU or CN available for {input_issuer}")
for pattern in issuer_patterns:
if pattern.search(match_name):
# Add a comment with the O and CN names if requested.
if comment:
out_.write(f"# O={org_name}, CN={common_name}\n".encode("ascii"))
out_.write(cert.public_bytes(Encoding.PEM))
if __name__ == "__main__":
run()

16
tools/issuers.txt Normal file
View file

@ -0,0 +1,16 @@
# These are Python regular expressions, one per line.
# Capitalization is ignored when doing matches.
Amazon
Baltimore
Comodo
Cybertrust
DigiCert
Digital Signature Trust
Entrust
GlobalSign
Go ?Daddy
Google Trust Services
Internet Security Research Group|ISRG
Starfield Technologies
USERTRUST
VeriSign

43
tools/test_site_coverage.py Executable file
View file

@ -0,0 +1,43 @@
#!/usr/bin/env python3
# SPDX-FileCopyrightText: 2023 Dan Halbert for Adafruit Industries LLC
#
# SPDX-License-Identifier: MIT
import click
import requests
from requests.exceptions import SSLError, RequestException
@click.command()
@click.option(
"--certs",
help="certificate bundle (.pem file)",
default="roots.pem",
type=click.Path(exists=True, dir_okay=False),
show_default=True,
)
@click.option(
"--urls",
help="file of URLs to test against the supplied certificate bundle",
default="urls.txt",
type=click.File("r"),
show_default=True,
)
def run(certs, urls):
for url in urls.readlines():
url = url.strip()
if not url or url.startswith("#"):
continue
try:
requests.request("GET", url, verify=certs, allow_redirects=True, timeout=20)
print("PASS", url)
except SSLError:
# Could not connect with given certs.
print("FAIL", url)
except RequestException as exc:
# Some other problem, unrelated to SSL issues.
print("SKIP", url, exc)
if __name__ == "__main__":
run()

495
tools/urls.txt Normal file
View file

@ -0,0 +1,495 @@
# These are problematic, with various connection failures, even though they work with a browser.
# The failures don't seem to be cert-related.
#https://www.iso.org
#https://www.nokia.com
#https://www.tripadvisor.com
#https://www.usnews.com
# bund.de uses D-Trust, which is rare
#https://bund.de
#CFCA (China Financial Certification Authority) cert, which is rare.
#https://english.news.cn
# Uses a CyberTrust Japan cert which is not in curl list (but works in browsers?)
#https://www.goo.ne.jp
#https://www.yahoo.co.jp
# error:0A000152:SSL routines::unsafe legacy renegotiation disabled
#https://oecd.org
# Uses Actalis, which is rare.
#https://huawei.com
https://000webhost.com
https://20minutos.es
https://4shared.com
https://abc.es
https://abc.net.au
https://abcnews.go.com
https://about.com
https://about.me
https://academia.edu
https://adafruit.com
https://adafruit-circuit-python.s3.amazonaws.com/index.html
https://addtoany.com
https://adobe.com
https://adweek.com
https://afternic.com
https://airbnb.com
https://alexa.com
https://alibaba.com
https://aliexpress.com
https://allaboutcookies.org
https://amazon.ca
https://amazon.co.jp
https://amazon.co.uk
https://amazon.com
https://amazon.de
https://amazon.es
https://amazon.fr
https://amazon.in
https://android.com
https://aol.com
https://ap.org
https://apache.org
https://api.coindesk.com
https://api.hackster.io
https://api.open-meteo.com
https://api.spacexdata.com
https://api.thingiverse.com
https://apple.com
https://archive.org
https://arxiv.org
https://asus.com
https://axs.com
https://bandcamp.com
https://bbc.co.uk
https://bbc.com
https://berkeley.edu
https://biblegateway.com
https://biglobe.ne.jp
https://billboard.com
https://bing.com
https://bit.ly
https://bitly.com
https://blackberry.com
https://blogger.com
https://bloglovin.com
https://bloomberg.com
https://booking.com
https://books.google.com
https://box.com
https://bp1.blogger.com
https://brandbucket.com
https://britannica.com
https://businessinsider.com
https://businessinsider.com.au
https://buzzfeed.com
https://ca.gov
https://cam.ac.uk
https://cambridge.org
https://canada.ca
https://canva.com
https://cbc.ca
https://cbsnews.com
https://cdc.gov
https://certificationapi.oshwa.org
https://change.org
https://channel4.com
https://chicagotribune.com
https://cia.gov
https://cloudflare.com
https://cmu.edu
https://cnbc.com
https://cnet.com
https://cnn.com
https://code.google.com
https://columbia.edu
https://com.com
https://cornell.edu
https://corriere.it
https://coursera.org
https://cpanel.com
https://cpanel.net
https://creativecommons.org
https://dailymail.co.uk
https://dailymotion.com
https://dan.com
https://daum.net
https://de.wikipedia.org
https://debian.org
https://deezer.com
https://depositfiles.com
https://detik.com
https://developers.google.com
https://dictionary.com
https://digg.com
https://discord.com
https://discord.gg
https://disney.com
https://disqus.com
https://docs.google.com
https://doi.org
https://doubleclick.net
https://dreniq.com
https://dribbble.com
https://drive.google.com
https://dropbox.com
https://dw.com
https://e-monsite.com
https://ea.com
https://economist.com
https://ed.gov
https://eff.org
https://elmundo.es
https://elpais.com
https://en.wikipedia.org
https://en.wordpress.com
https://enable-javascript.com
https://engadget.com
https://epa.gov
https://es.wikipedia.org
https://espn.com
https://espn.go.com
https://etsy.com
https://europa.eu
https://eventbrite.com
https://evernote.com
https://example.org
https://express.co.uk
https://facebook.com
https://fandom.com
https://fastcompany.com
https://fb.com
https://fb.me
https://fda.gov
https://feedburner.com
https://feedburner.google.com
https://feedproxy.google.com
https://fifa.com
https://files.wordpress.com
https://finance.yahoo.com
https://forbes.com
https://forms.gle
https://fortune.com
https://foursquare.com
https://foxnews.com
https://fr.wikipedia.org
https://freepik.com
https://ft.com
https://github.com
https://gizmodo.com
https://globo.com
https://gmail.com
https://gnu.org
https://godaddy.com
https://gofundme.com
https://goo.gl
https://goodreads.com
https://google.ca
https://google.co.id
https://google.co.in
https://google.co.jp
https://google.co.uk
https://google.com
https://google.com.au
https://google.com.br
https://google.com.tw
https://google.de
https://google.es
https://google.fr
https://google.it
https://google.nl
https://google.pl
https://google.ru
https://www.googleapis.com
https://googleblog.com
https://gooyaabitemplates.com
https://gravatar.com
https://groups.google.com
https://groups.yahoo.com
https://gstatic.com
https://guardian.co.uk
https://harvard.edu
https://hatena.ne.jp
https://hbr.org
https://histats.com
https://hm.com
https://home.neustar
https://hosted.weblate.org
https://howstuffworks.com
https://hp.com
https://huffingtonpost.com
https://huffpost.com
https://ibm.com
https://id.wikipedia.org
https://ietf.org
https://ig.com.br
https://ign.com
https://ikea.com
https://imageshack.com
https://imageshack.us
https://imdb.com
https://imgur.com
https://inc.com
https://independent.co.uk
https://indiatimes.com
https://insider.com
https://instagram.com
https://instructables.com
https://investopedia.com
https://io.adafruit.com
https://ipv4.google.com
https://irs.gov
https://issuu.com
https://istockphoto.com
https://it.wikipedia.org
https://iubenda.com
https://ja.wikipedia.org
https://jhu.edu
https://jstor.org
https://kickstarter.com
https://kotaku.com
https://latimes.com
https://lefigaro.fr
https://lemonde.fr
https://line.me
https://linkedin.com
https://live.com
https://loc.gov
https://lonelyplanet.com
https://lycos.com
https://m.wikipedia.org
https://mail.google.com
https://mail.ru
https://maps.google.com
https://marketingplatform.google.com
https://marketwatch.com
https://mashable.com
https://mediafire.com
https://medium.com
https://merriam-webster.com
https://metro.co.uk
https://microsoft.com
https://mirror.co.uk
https://mit.edu
https://mixcloud.com
https://mozilla.com
https://mozilla.org
https://msn.com
https://my.yahoo.com
https://myaccount.google.com
https://myspace.com
https://mysql.com
https://mystrikingly.com
https://namecheap.com
https://narod.ru
https://nationalgeographic.com
https://nature.com
https://naver.com
https://nbcnews.com
https://netflix.com
https://netvibes.com
https://networkadvertising.org
https://news.com.au
https://news.google.com
https://news.yahoo.com
https://newsweek.com
https://newyorker.com
https://nginx.com
https://nginx.org
https://nicovideo.jp
https://nikkei.com
https://noaa.gov
https://npr.org
https://nvidia.com
https://nydailynews.com
https://nypost.com
https://nytimes.com
https://office.com
https://offset.com
https://ok.ru
https://openaccess-api.clevelandart.org
https://opera.com
https://oracle.com
https://orange.fr
https://orkut.com.br
https://oup.com
https://over-blog-kiwi.com
https://ovh.co.uk
https://ovh.com
https://ovh.net
https://parallels.com
https://paypal.com
https://pbs.org
https://pcmag.com
https://pexels.com
https://photobucket.com
https://photos.google.com
https://php.net
https://picasa.google.com
https://picasaweb.google.com
https://pinterest.com
https://pixabay.com
https://pl.wikipedia.org
https://play.google.com
https://www.playstation.com
https://plesk.com
https://plos.org
https://politico.com
https://prestashop.com
https://prezi.com
https://princeton.edu
https://prnewswire.com
https://psu.edu
https://psychologytoday.com
https://pt.wikipedia.org
https://qq.com
https://quora.com
https://rakuten.co.jp
https://rambler.ru
https://rediff.com
https://repubblica.it
https://researchgate.net
https://reuters.com
https://reverbnation.com
https://ria.ru
https://rollingstone.com
https://rottentomatoes.com
https://rt.com
https://ru.wikipedia.org
https://samsung.com
https://sapo.pt
https://sciencedaily.com
https://sciencedirect.com
https://sciencemag.org
https://scientificamerican.com
https://scribd.com
https://search.google.com
https://search.yahoo.com
https://secureserver.net
https://sedo.com
https://sendspace.com
https://sfgate.com
https://shopify.com
https://shutterstock.com
https://si.edu
https://sina.com.cn
https://sites.google.com
https://sky.com
https://skype.com
https://slate.com
https://slideshare.net
https://smh.com.au
https://snapchat.com
https://so-net.ne.jp
https://softonic.com
https://softpedia.com
https://soratemplates.com
https://soundcloud.com
https://spiegel.de
https://sports.yahoo.com
https://spotify.com
https://springer.com
https://sputniknews.com
https://stackoverflow.com
https://standard.co.uk
https://stanford.edu
https://statista.com
https://steamcommunity.com
https://steampowered.com
https://storage.canalblog.com
https://storage.googleapis.com
https://support.google.com
https://surveymonkey.com
https://t.co
https://t.me
https://target.com
https://techcrunch.com
https://techradar.com
https://ted.com
https://telegram.me
https://telegraph.co.uk
https://terra.com.br
https://theatlantic.com
https://thefreedictionary.com
https://theglobeandmail.com
https://theguardian.com
https://themeforest.net
https://thenextweb.com
https://thestar.com
https://thesun.co.uk
https://thetimes.co.uk
https://theverge.com
https://thoughtco.com
https://time.com
https://timeout.com
https://tinyurl.com
https://tools.google.com
https://translate.google.com
https://trustpilot.com
https://twitch.tv
https://twitter.com
https://ubuntu.com
https://ucoz.ru
https://umich.edu
https://unicef.org
https://uol.com.br
https://urbandictionary.com
https://usatoday.com
https://usgs.gov
https://utexas.edu
https://variety.com
https://venturebeat.com
https://viglink.com
https://vimeo.com
https://vk.com
https://vkontakte.ru
https://w3.org
https://wa.me
https://walmart.com
https://washington.edu
https://washingtonpost.com
https://weather.com
https://webmd.com
https://weibo.com
https://welt.de
https://whatsapp.com
https://whitehouse.gov
https://who.int
https://wikia.com
https://wikihow.com
https://wikimedia.org
https://wiktionary.org
https://wiley.com
https://wired.com
https://wisc.edu
https://wn.com
https://wordpress.org
https://worldbank.org
https://wp.com
https://wsj.com
https://www.canalblog.com
https://www.ebay.com
https://www.gov.uk
https://www.icann.org
https://www.livejournal.com
https://www.nasa.gov
https://www.nih.gov
https://www.over-blog.com
https://www.un.org
https://www.unesco.org
https://www.weebly.com
https://www.wikipedia.org
https://www.wix.com
https://www.yahoo.com
https://xbox.com
https://yadi.sk
https://yale.edu
https://yandex.ru
https://yelp.com
https://youtube.com
https://zeit.de
https://zendesk.com
https://ziddu.com