When istio proxy comes up for the first time, and it is given certificates via SDS, it was observed that it stores duplicate entries of the certificate as seen in the /certs endpoint.
Even when certificates rotate, the count of the original certificate remains consistent in /certs endpoint.
The above command show that there are 29 instances of cert_chain, even though the serial number of all the certificates are the same. Which means that /certs endpoint is storing duplicate entries.
{
"certificates": [
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
},
{
"ca_cert": [
{
"path": "\u003cinline\u003e",
"serial_number": "e2110e139cc6cc7a",
"subject_alt_names": [],
"days_until_expiration": "34863",
"valid_from": "2018-01-24T19:15:51Z",
"expiration_time": "2117-12-31T19:15:51Z"
}
],
"cert_chain": [
{
"path": "\u003cinline\u003e",
"serial_number": "26931ad6c653ea029212660e6edb781c",
"subject_alt_names": [
{
"uri": "spiffe://cluster.local/ns/bookinfo/sa/bookinfo-ratings"
}
],
"days_until_expiration": "0",
"valid_from": "2022-07-19T05:18:54Z",
"expiration_time": "2022-07-20T05:20:54Z"
}
]
}
]
}
Bug Description
Issue 1
When istio proxy comes up for the first time, and it is given certificates via SDS, it was observed that it stores duplicate entries of the certificate as seen in the
/certsendpoint.Issue 2
Envoy continues to listen on stale certificates, even when
/certsendpoint shows that it has new certificates. One thing to note here is, that the/certsendpoint shows both new as well as old certificates.Envoy is serving stale certificates only when there are more than 1 unique certificates found in the
/certsendpoint.Version
Additional data points
It is consistently seen that the
/certsendpoints shows constant occurrences of the old certificate:Even when certificates rotate, the count of the original certificate remains consistent in
/certsendpoint.Some parsed data from the
/certsendpoint:The above command show that there are 29 instances of cert_chain, even though the serial number of all the certificates are the same. Which means that
/certsendpoint is storing duplicate entries.Data from
/certsendpoint in cluster created via kind.