Smallstep CA and Caddy#

Motivation

zero trust networking
ACME server, ACME client
auto TLS
client certificates (mTLS)

Sources

Forgot Caddy Volume#

If you forgot to add a volume for caddy at /data, then you likely get the following error messages:

Client error messages like

{"level":"info","ts":1647011080.4116142,"logger":"tls.issuance.acme","msg":"waiting on internal rate limiter","identifiers":["node.runner1.example.blue"],"ca":"https://ca.example.red/acme/acme/directory","account":"alice@example.com"}
{"level":"info","ts":1647011080.4116533,"logger":"tls.issuance.acme","msg":"done waiting on internal rate limiter","identifiers":["node.runner1.example.blue"],"ca":"https://ca.example.red/acme/acme/directory","account":"alice@example.com"}
{"level":"info","ts":1647011080.441212,"logger":"tls.issuance.acme.acme_client","msg":"trying to solve challenge","identifier":"node.runner1.example.blue","challenge_type":"http-01","ca":"https://ca.example.red/acme/acme/directory"}
{"level":"warn","ts":1647011095.4436758,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011110.962895,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011126.478382,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011141.9942307,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011157.5120714,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011173.0251896,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011188.5408232,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011204.053496,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011219.570125,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": http2: timeout awaiting response headers"}
{"level":"warn","ts":1647011235.0838146,"logger":"tls.issuance.acme.acme_client","msg":"HTTP request failed; retrying","url":"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ","error":"performing request: Post \"https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ\": stream error: stream ID 47; INTERNAL_ERROR"}
{"level":"error","ts":1647011235.3580816,"logger":"tls.issuance.acme.acme_client","msg":"deactivating authorization","identifier":"node.runner1.example.blue","authz":"https://ca.example.red/acme/acme/authz/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE","error":"attempt 1: https://ca.example.red/acme/acme/authz/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE: HTTP 0 urn:ietf:params:acme:error:malformed - The request message was malformed"}
{"level":"error","ts":1647011235.358188,"logger":"tls.issuance.acme.acme_client","msg":"validating authorization","problem":{"type":"urn:ietf:params:acme:error:badNonce","title":"","detail":"Unacceptable anti-replay nonce","instance":"","subproblems":[]},"order":"https://ca.example.red/acme/acme/order/BfHBdOEvHmN3HSqy17zxkk6BBjZhgor6","attempt":1,"max_attempts":3}
{"level":"error","ts":1647011235.3582716,"logger":"tls.obtain","msg":"could not get certificate from issuer","identifier":"node.runner1.example.blue","issuer":"ca.example.red-acme-acme-directory","error":"HTTP 0 urn:ietf:params:acme:error:badNonce - Unacceptable anti-replay nonce"}
{"level":"error","ts":1647011235.3583107,"logger":"tls.obtain","msg":"will retry","error":"[node.runner1.example.blue] Obtain: [node.runner1.example.blue] solving challenge: initiating challenge with server: attempt 11: https://ca.example.red/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ: HTTP 0 urn:ietf:params:acme:error:badNonce - Unacceptable anti-replay nonce (ca=https://ca.example.red/acme/acme/directory)","attempt":1,"retrying_in":60,"elapsed":155.02513457,"max_duration":2592000}

CA error messages like

time="2022-03-11T15:04:55Z" level=warning duration=5.853879ms duration-ns=5853879 error="nonce cnVCNU1Jem90cG9lUXBHSk56NmQydUpuWTBXRkNNdHE not found" fields.time="2022-03-11T15:04:55Z" method=POST name=ca nonce=RkdhdFpkTVp3SFlZeFpyT05WUlhVVTZCZmxQaUh5T0Q path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lma5o14t6j2kkcas40 size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:05:11Z" level=warning duration=4.254932ms duration-ns=4254932 error="nonce RkdhdFpkTVp3SFlZeFpyT05WUlhVVTZCZmxQaUh5T0Q not found" fields.time="2022-03-11T15:05:11Z" method=POST name=ca nonce=QVBTbklBdWw2WkliZVlJSVVwWVdLMW5uS05vU1lpVng path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lma9o14t6j2kkcas5g size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:05:26Z" level=warning duration=4.66413ms duration-ns=4664130 error="nonce QVBTbklBdWw2WkliZVlJSVVwWVdLMW5uS05vU1lpVng not found" fields.time="2022-03-11T15:05:26Z" method=POST name=ca nonce=ZDhsRVY0ZWVoVUF6SDJldWEwQnpuSW9WSjEwSHNuc08 path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmadg14t6j2kkcas6g size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:05:42Z" level=warning duration=3.562891ms duration-ns=3562891 error="nonce ZDhsRVY0ZWVoVUF6SDJldWEwQnpuSW9WSjEwSHNuc08 not found" fields.time="2022-03-11T15:05:42Z" method=POST name=ca nonce=ejRMeHpYN2Rjb1VXeXVJYk1iWTlmMHV2V1htSVJMUFI path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmahg14t6j2kkcas80 size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:05:57Z" level=warning duration=2.640927ms duration-ns=2640927 error="nonce ejRMeHpYN2Rjb1VXeXVJYk1iWTlmMHV2V1htSVJMUFI not found" fields.time="2022-03-11T15:05:57Z" method=POST name=ca nonce=Mnl4V1VhaUJBNmIyTENsclFGWjJtZWNFTHozY3FCNGU path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmal814t6j2kkcas90 size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:06:13Z" level=warning duration=3.076647ms duration-ns=3076647 error="nonce Mnl4V1VhaUJBNmIyTENsclFGWjJtZWNFTHozY3FCNGU not found" fields.time="2022-03-11T15:06:13Z" method=POST name=ca nonce=RDVPZ1lZM1ZxR3ZqdVB5cWpTMlVwWkJrTEJOQUpYbUw path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmap814t6j2kkcasag size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:06:28Z" level=warning duration=3.004316ms duration-ns=3004316 error="nonce RDVPZ1lZM1ZxR3ZqdVB5cWpTMlVwWkJrTEJOQUpYbUw not found" fields.time="2022-03-11T15:06:28Z" method=POST name=ca nonce=M0dPTmpTclJERjQ5dGRBdHhhY2RKRGxsVk5ubXhpMnk path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmat014t6j2kkcasbg size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:06:44Z" level=warning duration=3.321361ms duration-ns=3321361 error="nonce M0dPTmpTclJERjQ5dGRBdHhhY2RKRGxsVk5ubXhpMnk not found" fields.time="2022-03-11T15:06:44Z" method=POST name=ca nonce=ZlN2VHdIOFlmWXNaSGpVVzRlVnBlRHd6d2R0Y3l4RGk path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmb1014t6j2kkcasd0 size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:06:59Z" level=warning duration=2.467454ms duration-ns=2467454 error="nonce ZlN2VHdIOFlmWXNaSGpVVzRlVnBlRHd6d2R0Y3l4RGk not found" fields.time="2022-03-11T15:06:59Z" method=POST name=ca nonce=bnhpOEh1RnlBUXJPb3cxdGVUNktOWUFZTEVvMUVtckw path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmb4o14t6j2kkcase0 size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:07:15Z" level=warning duration=2.186935ms duration-ns=2186935 error="nonce bnhpOEh1RnlBUXJPb3cxdGVUNktOWUFZTEVvMUVtckw not found" fields.time="2022-03-11T15:07:15Z" method=POST name=ca nonce=T0Z1R1pOYXhiMUtjaWVnV2dFdTBWQlNUbmRhNWhlZDA path=/acme/acme/challenge/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE/fDJcw8SldJHpb7zAfUwMqiGdNYSDkdPJ protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmb8o14t6j2kkcasfg size=89 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=
time="2022-03-11T15:07:15Z" level=warning duration=10.063699ms duration-ns=10063699 error="expected POST-as-GET" fields.time="2022-03-11T15:07:15Z" method=POST name=ca nonce=MTBDcWlmZlJ6dW5wZVZteGFmc3FDREtzZzlwaDFXdlg path=/acme/acme/authz/Bz578KV3nZYSQaPI9QQ9d43aJ1TyrLhE protocol=HTTP/2.0 referer= remote-address=123.123.123.123 request-id=c8lmb8o14t6j2kkcasg0 size=93 status=400 user-agent="Caddy/2.4.6 CertMagic acmez (linux; amd64)" user-id=

FIX (docker-compose.yml)

volumes:
  caddy_data:

services:
  caddy:
    image: caddy:2.4.6
    volumes:
      - caddy_data:/data
      - /var/log/caddy:/var/log/caddy
    ports:
      - "443:443"
    restart: unless-stopped

The Documentation System apt-get update EXPKEYSIG

Neue Einträge

Schlagworte

Archive

Sprachen

Smallstep CA and Caddy#

Forgot Caddy Volume#