1 package entrypoint 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path" 9 "time" 10 11 "github.com/datawire/dlib/dlog" 12 "github.com/emissary-ingress/emissary/v3/pkg/kates" 13 "github.com/emissary-ingress/emissary/v3/pkg/snapshot/v1" 14 snapshotTypes "github.com/emissary-ingress/emissary/v3/pkg/snapshot/v1" 15 ) 16 17 // The IstioCertSource and IstioCertWatcher interfaces exist to allow dependency 18 // injection while testing the watcher. What you see here are the production 19 // implementations: 20 // 21 // istioCertSource implements IstioCertSource: its Watch() method returns an 22 // istioCertWatcher, which implements IstioCertWatcher in turn. 23 type istioCertSource struct { 24 } 25 26 type istioCertWatcher struct { 27 updateChannel chan IstioCertUpdate 28 } 29 30 func newIstioCertSource() IstioCertSource { 31 return &istioCertSource{} 32 } 33 34 // Watch sets up to watch for an Istio cert on the filesystem, if need be. This 35 // is the production implementation, which returns an istioCertWatcher to implement 36 // the IstioCertWatcher interface. 37 func (src *istioCertSource) Watch(ctx context.Context) (IstioCertWatcher, error) { 38 // We can watch the filesystem for Istio mTLS certificates. Here, we fire 39 // up the stuff we need to do that -- specifically, we need an FSWatcher 40 // to watch the filesystem, an IstioCert to manage the cert, and an update 41 // channel to hear about new Istio stuff. 42 // 43 // The actual functionality here is currently keyed off the environment 44 // variable AMBASSADOR_ISTIO_SECRET_DIR, but we set the update channel 45 // either way to keep the select logic below simpler. If the environment 46 // variable is unset, we never instantiate the FSWatcher or IstioCert, 47 // so there will never be any updates on the update channel. 48 istioCertUpdateChannel := make(chan IstioCertUpdate) 49 50 // OK. Are we supposed to watch anything? 51 secretDir := os.Getenv("AMBASSADOR_ISTIO_SECRET_DIR") 52 53 if secretDir != "" { 54 // Yup, get to it. First, fire up the IstioCert, and tell it to 55 // post to our update channel from above. 56 icert := NewIstioCert(secretDir, "istio-certs", GetAmbassadorNamespace(), istioCertUpdateChannel) 57 58 // Next up, fire up the FSWatcher... 59 fsw, err := NewFSWatcher(ctx) 60 if err != nil { 61 return nil, err 62 } 63 go fsw.Run(ctx) 64 65 // ...then tell the FSWatcher to watch the Istio cert directory, 66 // and give it a handler function that'll update the IstioCert 67 // in turn. 68 // 69 // XXX This handler function is really just an impedance matcher. 70 // Maybe IstioCert should just have a "HandleFSWEvent"... 71 err = fsw.WatchDir(ctx, secretDir, 72 func(ctx context.Context, event FSWEvent) { 73 // Is this a deletion? 74 deleted := (event.Op == FSWDelete) 75 76 // OK. Feed this event into the IstioCert. 77 icert.HandleEvent(ctx, event.Path, deleted) 78 }, 79 ) 80 if err != nil { 81 dlog.Errorf(ctx, "FileSystemWatcher.WatchDir(ctx, %q, fn) => %v", 82 secretDir, err) 83 } 84 } 85 86 return &istioCertWatcher{ 87 updateChannel: istioCertUpdateChannel, 88 }, nil 89 } 90 91 // Changed returns the channel where Istio certificates will appear. 92 func (istio *istioCertWatcher) Changed() <-chan IstioCertUpdate { 93 return istio.updateChannel 94 } 95 96 // istioCertWatchManager is the interface between all the Istio-cert-watching stuff 97 // and the watcher (in watcher.go). 98 type istioCertWatchManager struct { 99 // XXX Temporary hack: we currently store the secrets found by the Istio-cert 100 // watcher in the K8s snapshot, but this gives the Istio-cert watcher an easy 101 // way to note that it saw changes. This is important because if any of the 102 // watchers see changes, we can't short-circuit the reconfiguration. 103 watcher IstioCertWatcher 104 changesPresent bool 105 } 106 107 // Changed returns a channel to listen on for change notifications dealing with 108 // Istio cert stuff. 109 func (imgr *istioCertWatchManager) Changed() <-chan IstioCertUpdate { 110 return imgr.watcher.Changed() 111 } 112 113 // Update actually does the work of updating our internal state with changes. The 114 // istioCertWatchManager isn't allowed to short-circuit early: it's assumed that 115 // any update is relevant. 116 func (imgr *istioCertWatchManager) Update(ctx context.Context, icertUpdate IstioCertUpdate, k8sSnapshot *snapshot.KubernetesSnapshot) { 117 dlog.Debugf(ctx, "WATCHER: ICert fired") 118 119 // We've seen a change in the Istio cert info on the filesystem. This is 120 // kind of a hack, but let's just go ahead and say that if we see an event 121 // here, it's a real change -- presumably we won't be told to watch Istio 122 // certs if they aren't important. 123 // 124 // XXX Obviously this is a crock and we should actually track whether the 125 // secret is in use. 126 imgr.changesPresent = true 127 128 // Make a SecretRef for this new secret... 129 ref := snapshotTypes.SecretRef{Name: icertUpdate.Name, Namespace: icertUpdate.Namespace} 130 131 // ...and delete or save, as appropriate. 132 if icertUpdate.Op == "delete" { 133 dlog.Infof(ctx, "IstioCert: certificate %s.%s deleted", icertUpdate.Name, icertUpdate.Namespace) 134 delete(k8sSnapshot.FSSecrets, ref) 135 } else { 136 dlog.Infof(ctx, "IstioCert: certificate %s.%s updated", icertUpdate.Name, icertUpdate.Namespace) 137 k8sSnapshot.FSSecrets[ref] = icertUpdate.Secret 138 } 139 // Once done here, k8sSnapshot.ReconcileSecrets will handle the rest. 140 } 141 142 // StartLoop sets up the istioCertWatchManager for the start of the watcher loop. 143 func (imgr *istioCertWatchManager) StartLoop(ctx context.Context) { 144 // Start every loop by assuming that no changes are present. 145 imgr.changesPresent = false 146 } 147 148 // UpdatesPresent returns whether or not any significant updates have actually 149 // happened. 150 func (imgr *istioCertWatchManager) UpdatesPresent() bool { 151 return imgr.changesPresent 152 } 153 154 // newIstioCertWatchManager returns... a new istioCertWatchManager. 155 func newIstioCertWatchManager(ctx context.Context, watcher IstioCertWatcher) *istioCertWatchManager { 156 istio := istioCertWatchManager{ 157 watcher: watcher, 158 changesPresent: false, 159 } 160 161 return &istio 162 } 163 164 // Istio TLS certificates are annoying. They come in three parts (only two of 165 // which are needed), they're updated non-atomically, and we need to make sure we 166 // don't try to reconfigure when the parts are out of sync. Therefore, we keep 167 // track of the last-update time of both parts, and only update once both have 168 // been updated at the "same" time. 169 170 type pemReader func(ctx context.Context, dir string, name string) ([]byte, bool) 171 type timeFetcher func() time.Time 172 173 // IstioCert holds all the state we need to manage an Istio certificate. 174 type IstioCert struct { 175 dir string 176 name string // Name we'll use when generating our secret 177 namespace string // Namespace in which our secret will appear to be 178 timestamps map[string]time.Time 179 180 // How shall we read PEM files? 181 readPEM pemReader 182 183 // How shall we fetch the current time? 184 fetchTime timeFetcher 185 186 // Where shall we send updates when things happen? 187 updates chan IstioCertUpdate 188 } 189 190 // IstioCertUpdate gets sent over the IstioCert's Updates channel 191 // whenever the cert changes 192 // 193 // XXX This will morph into a more general "internally-generated resource" 194 // thing later. 195 type IstioCertUpdate struct { 196 Op string // "update" or "delete" 197 Name string // secret name 198 Namespace string // secret namespace 199 Secret *kates.Secret // IstioCert secret 200 } 201 202 // NewIstioCert instantiates an IstioCert to manage a certificate that Istio 203 // will write into directory "dir", should have the given "name" and appear 204 // to live in K8s namespace "namespace", and will have updates posted to 205 // "updateChannel" whenever the cert changes. 206 // 207 // What's with this namespace business? Well, Ambassador may be running in 208 // single-namespace mode, so causing our cert to appear to be in the same 209 // namespace as Ambassador will just be less confusing for everyone. 210 // 211 // XXX Nomenclature is a little odd here. Istio is writing a _certificate_, 212 // but we're supplying it to the rest of Ambassador as a thing that looks 213 // like a Kubernetes TLS _Secret_ -- so we call this class an IstioCert, 214 // but the thing it's posting to the updateChannel includes a kates.Secret. 215 // Names are hard. 216 func NewIstioCert(dir string, name string, namespace string, updateChannel chan IstioCertUpdate) *IstioCert { 217 icert := &IstioCert{ 218 dir: dir, 219 name: name, 220 namespace: namespace, 221 fetchTime: time.Now, // default to using time.Now for time 222 updates: updateChannel, 223 } 224 225 // Default to using our default PEM reader... 226 icert.readPEM = icert.defaultReadPEM 227 228 // ...initialize the timestamp map... 229 icert.timestamps = make(map[string]time.Time) 230 231 return icert 232 } 233 234 // String returns a string representation of this IstioCert. 235 func (icert *IstioCert) String() string { 236 // Our dir may be nothing, if we're just a dummy IstioCert 237 // that's being used to make other logic easier. If that's the 238 // case, be a little more verbose here. 239 240 if icert.dir == "" { 241 return "IstioCert (noop)" 242 } 243 244 return fmt.Sprintf("IstioCert %s", icert.dir) 245 } 246 247 // SetFetchTime will change the function we use to get the current time. 248 func (icert *IstioCert) SetFetchTime(fetchTime timeFetcher) { 249 icert.fetchTime = fetchTime 250 } 251 252 // SetReadPEM will change the function we use to read PEM files. 253 func (icert *IstioCert) SetReadPEM(readPEM pemReader) { 254 icert.readPEM = readPEM 255 } 256 257 // defaultReadPEM is the same as ioutil.ReadFile, really, but it logs for us 258 // if anything goes wrong. 259 func (icert *IstioCert) defaultReadPEM(ctx context.Context, dir string, name string) ([]byte, bool) { 260 pemPath := path.Join(dir, name) 261 262 pem, err := ioutil.ReadFile(pemPath) 263 264 if err != nil { 265 dlog.Errorf(ctx, "%s: couldn't read %s: %s", icert, pemPath, err) 266 return nil, false 267 } 268 269 return pem, true 270 } 271 272 // getTimeFor tries to look up the timestamp we have stored for a given key, 273 // but it logs if it's missing (for debugging). 274 func (icert *IstioCert) getTimeFor(ctx context.Context, name string) (time.Time, bool) { 275 then, exists := icert.timestamps[name] 276 277 if !exists { 278 dlog.Debugf(ctx, "%s: %s missing", icert, name) 279 return time.Time{}, false 280 } 281 282 return then, true 283 } 284 285 // HandleEvent tells an IstioCert to update its internal state because a file 286 // in its directory has been written. If all the cert files have been updated 287 // closely enough in time, Update will decide that it's time to actually update 288 // the cert, and it'll send an IstioCertUpdate over the Updates channel. 289 func (icert *IstioCert) HandleEvent(ctx context.Context, name string, deleted bool) { 290 // Istio writes three files into its cert directory: 291 // - key.pem is the private key 292 // - cert-chain.pem is the public keychain 293 // - root-cert.pem is the CA root public key 294 // 295 // We ignore root-cert.pem, because cert-chain.pem contains it, and we 296 // ignore any other name because Istio shouldn't be writing it there. 297 // 298 // Start by splitting the incoming name (which is really a path) into its 299 // component parts, just 'cause it (mostly) makes life easier to refer 300 // to things by the basename (the key) rather than the full path. 301 dir := path.Dir(name) 302 key := path.Base(name) 303 304 dlog.Debugf(ctx, "%s: updating %s at %s", icert, key, name) 305 306 if dir != icert.dir { 307 // Somehow this is in the wrong directory? Toss it. 308 dlog.Debugf(ctx, "%s: ignoring %s in dir %s", icert, name, dir) 309 return 310 } 311 312 if (key != "key.pem") && (key != "cert-chain.pem") { 313 // Someone is writing a file we don't need. Toss it. 314 dlog.Debugf(ctx, "%s: ignoring %s", icert, name) 315 return 316 } 317 318 // If this is a deletion... 319 if deleted { 320 // ...then drop the key from our timestamps map. 321 delete(icert.timestamps, key) 322 } else { 323 // Not a deletion -- update the time for this key. 324 icert.timestamps[key] = icert.fetchTime() 325 } 326 327 // Do we have both key.pem and cert-chain.pem? (It's OK to just return immediately 328 // without logging, if not, because getTime logs for us.) 329 kTime, kExists := icert.getTimeFor(ctx, "key.pem") 330 cTime, cExists := icert.getTimeFor(ctx, "cert-chain.pem") 331 332 bothPresent := (kExists && cExists) 333 334 // OK. Is it time to do anything? 335 if bothPresent && !deleted { 336 // Maybe. Everything we need is present, but are the updates close enough 337 // in time? Start by finding out which of key & cert-chain is newest, so 338 // that we can find the delta between them. 339 // 340 // XXX Wouldn't it be nice if time.Duration had an AbsValue method?t 341 var delta time.Duration 342 343 if cTime.Before(kTime) { 344 delta = kTime.Sub(cTime) 345 } else { 346 delta = cTime.Sub(kTime) 347 } 348 349 // OK, if the delta is more than five seconds (which is crazy long), we're done. 350 // 351 // Why five? Well, mostly just 'cause it's really easy to imagine these getting 352 // written on different sides of a second boundary, really hard to imagine it 353 // taking longer than five seconds, and really hard to imagine trying to rotate 354 // certs every few seconds instead of every few minutes... 355 356 if delta > 5*time.Second { 357 dlog.Debugf(ctx, "%s: cert-chain/key delta %v, out of sync", icert, delta) 358 return 359 } 360 361 // OK, times look good -- grab the JSON for this thing. 362 secret, ok := icert.Secret(ctx) 363 364 if !ok { 365 // WTF. 366 dlog.Debugf(ctx, "%s: cannot construct secret", icert) 367 } 368 369 // FINALLY! 370 dlog.Debugf(ctx, "%s: noting update!", icert) 371 372 go func() { 373 icert.updates <- IstioCertUpdate{ 374 Op: "update", 375 Name: secret.ObjectMeta.Name, 376 Namespace: secret.ObjectMeta.Namespace, 377 Secret: secret, 378 } 379 380 dlog.Debugf(ctx, "%s: noted update!", icert) 381 }() 382 } else if deleted && !bothPresent { 383 // OK, this is a deletion, and we no longer have both files. Time to 384 // post the deletion. 385 // 386 // XXX We can actually generate _two_ deletions if both files are 387 // deleted. We're not worrying about that for now. 388 389 dlog.Debugf(ctx, "%s: noting deletion", icert) 390 391 go func() { 392 // Kind of a hack -- since we can't generate a real Secret object 393 // without having the files we need, send the name & namespace from 394 // icert. 395 icert.updates <- IstioCertUpdate{ 396 Op: "delete", 397 Name: icert.name, 398 Namespace: icert.namespace, 399 Secret: nil, 400 } 401 402 dlog.Debugf(ctx, "%s: noted deletion!", icert) 403 }() 404 } else { 405 dlog.Debugf(ctx, "%s: nothing to note", icert) 406 } 407 } 408 409 // Secret generates a kates.Secret for this IstioCert. Since this 410 // involves reading PEM, it can fail, so it logs and returns a status. 411 func (icert *IstioCert) Secret(ctx context.Context) (*kates.Secret, bool) { 412 privatePEM, privateOK := icert.readPEM(ctx, icert.dir, "key.pem") 413 publicPEM, publicOK := icert.readPEM(ctx, icert.dir, "cert-chain.pem") 414 415 if !privateOK || !publicOK { 416 dlog.Errorf(ctx, "%s: read error, bailing", icert) 417 return nil, false 418 } 419 420 newSecret := &kates.Secret{ 421 TypeMeta: kates.TypeMeta{ 422 APIVersion: "v1", 423 Kind: "Secret", 424 }, 425 ObjectMeta: kates.ObjectMeta{ 426 Name: icert.name, 427 Namespace: icert.namespace, 428 }, 429 Type: kates.SecretTypeTLS, 430 Data: map[string][]byte{ 431 "tls.key": privatePEM, 432 "tls.crt": publicPEM, 433 }, 434 } 435 436 return newSecret, true 437 } 438