{"page":{"id":"01K0Q1X4P7X87YAH6FHH452BES","name":"turbopuffer","url":"https://status.turbopuffer.com/","updated_at":"2026-09-05T18:52:40Z"},"incidents":[{"id":"01M1SDXV1KRRFS61H6HMXPVA0P","name":"Dashboard outage","status":"resolved","created_at":"2026-09-05T18:40:07Z","updated_at":"2026-09-05T18:52:40Z","resolved_at":"2026-09-05T18:52:40Z","impact":"critical","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01M1SEMTCYZ37W0100B5VGT8HH","body":"From 7:21-7:41 UTC, the turbopuffer dashboard was unavailable due to an issue with our backing datastore. This issue has been resolved and dashboard access has been restored.","created_at":"2026-09-05T18:52:40Z","display_at":"2026-09-05T18:52:40Z","incident_id":"01M1SDXV1KRRFS61H6HMXPVA0P","status":"resolved","updated_at":"2026-09-05T18:52:40Z"},{"id":"01M1SDXV1K7G9CKYZDPEFE4CBB","body":"The turbopuffer dashboard is currently experiencing an outage. turbopuffer regions are fully available. Writes and queries are not affected.","created_at":"2026-09-05T18:40:07Z","display_at":"2026-09-05T18:40:07Z","incident_id":"01M1SDXV1KRRFS61H6HMXPVA0P","status":"investigating","updated_at":"2026-09-05T18:40:07Z"}]},{"id":"01M1HGC7Z7PRG0EKR7D7V92SXH","name":"Elevated rate of 5xx in gcp-us-central1","status":"resolved","created_at":"2026-09-02T16:49:00Z","updated_at":"2026-09-02T17:45:24Z","resolved_at":"2026-09-02T17:16:00Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01M1HKKFV71S8777N6ZZTDM01S","body":"The issue causing elevated 5xx errors for some namespaces has been resolved. We repaired the affected namespaces, and error rates have returned to normal. We will continue monitoring the service closely.","created_at":"2026-09-02T17:45:24Z","display_at":"2026-09-02T17:16:00Z","incident_id":"01M1HGC7Z7PRG0EKR7D7V92SXH","status":"resolved","updated_at":"2026-09-02T17:45:38Z"},{"id":"01M1HGC7Z7X2H44TGHD9AD1SHY","body":"Some namespaces are experiencing an elevated rate of 5xx errors. We have identified the issue and are working to restore full service.","created_at":"2026-09-02T16:49:00Z","display_at":"2026-09-02T16:49:00Z","incident_id":"01M1HGC7Z7PRG0EKR7D7V92SXH","status":"monitoring","updated_at":"2026-09-02T16:49:00Z"}]},{"id":"01M12F2F12X8ZS91AEAY4BKGYS","name":"Outage in gcp-us-southeast1","status":"resolved","created_at":"2026-08-27T20:37:35Z","updated_at":"2026-08-27T22:15:13Z","resolved_at":"2026-08-27T22:15:13Z","impact":"critical","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01M12MN7K1NT9PVS5PH2K7PQ1F","body":"Service has been fully restored.","created_at":"2026-08-27T22:15:13Z","display_at":"2026-08-27T22:15:13Z","incident_id":"01M12F2F12X8ZS91AEAY4BKGYS","status":"resolved","updated_at":"2026-08-27T22:15:13Z"},{"id":"01M12GT9Q23YJA686KF2NEDA1Y","body":"We’ve identified the problem and rolled out a fix. We’re closely monitoring the situation.","created_at":"2026-08-27T21:08:05Z","display_at":"2026-08-27T21:08:05Z","incident_id":"01M12F2F12X8ZS91AEAY4BKGYS","status":"monitoring","updated_at":"2026-08-27T21:08:05Z"},{"id":"01M12F2F12RA7D7M0YWWDA4B5C","body":"Full outage identified in gcp-southeast1","created_at":"2026-08-27T20:37:35Z","display_at":"2026-08-27T20:37:35Z","incident_id":"01M12F2F12X8ZS91AEAY4BKGYS","status":"investigating","updated_at":"2026-08-27T20:37:35Z"}]},{"id":"01KXN4XEHJZQX6V2SK4QJZJ1TX","name":"Errors when accessing dashboard","status":"resolved","created_at":"2026-07-16T09:42:52Z","updated_at":"2026-07-16T10:49:22Z","resolved_at":"2026-07-16T10:49:22Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KXN8Q7PTYX8E33H01HVGSEAW","body":"Our authentication provider has resolved the incident on their end, and dashboard logins have been fully operational since 10:08 UTC.\n\n Existing sessions and the database were unaffected throughout.","created_at":"2026-07-16T10:49:22Z","display_at":"2026-07-16T10:49:22Z","incident_id":"01KXN4XEHJZQX6V2SK4QJZJ1TX","status":"resolved","updated_at":"2026-07-16T10:49:22Z"},{"id":"01KXN6Q1MNBBJWJ3EFM2F5QA35","body":"Logins are succeeding again as of 10:08 UTC. Our authentication provider is still working through their incident, so some logins may be slower than usual. \n\nWe'll continue monitoring until it's fully resolved.","created_at":"2026-07-16T10:14:19Z","display_at":"2026-07-16T10:14:19Z","incident_id":"01KXN4XEHJZQX6V2SK4QJZJ1TX","status":"monitoring","updated_at":"2026-07-16T10:14:19Z"},{"id":"01KXN67BJX8E89HSN3JWCBXM95","body":"Dashboard logins (Google, SSO, and email) are intermittently failing due to an ongoing incident at [our authentication provider](https://status.workos.com/incidents/b52fkqjzcl20). \n\n We're monitoring and will update here.","created_at":"2026-07-16T10:05:45Z","display_at":"2026-07-16T10:05:45Z","incident_id":"01KXN4XEHJZQX6V2SK4QJZJ1TX","status":"identified","updated_at":"2026-07-16T10:05:45Z"},{"id":"01KXN4XEHJV5XB8QYDXQS19MMH","body":"We're aware of issues affecting some customers who are logging into the dashboard. We're investigating and will provide an update shortly.\n\nNote that the database is not affected - this issue is limited to the dashboard.","created_at":"2026-07-16T09:42:52Z","display_at":"2026-07-16T09:42:52Z","incident_id":"01KXN4XEHJZQX6V2SK4QJZJ1TX","status":"investigating","updated_at":"2026-07-16T09:42:52Z"}]},{"id":"01KSQYW7MBTXMQDDKCWK8RERZ2","name":"Elevated 500 rate on delete_by_filter writes, filtered queries","status":"resolved","created_at":"2026-05-28T18:52:16Z","updated_at":"2026-05-28T18:54:48Z","resolved_at":"2026-05-28T18:30:00Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KSQZ0VWQC4FRMXG7YKG0CJ7F","body":"Starting at ~17:55 UTC, we experienced an elevated 500 rate affecting delete_by_filter, and filtered queries as a result of a recent change deployed to filtering. This change was reverted, and errors fully subsided by ~18:30 UTC. This incident affected approximately 38.2% of `delete_by_filter` requests in aws-us-east-1, and a small percentage of filtered queries in other regions.","created_at":"2026-05-28T18:54:48Z","display_at":"2026-05-28T18:30:00Z","incident_id":"01KSQYW7MBTXMQDDKCWK8RERZ2","status":"resolved","updated_at":"2026-05-28T19:01:09Z"},{"id":"01KSQYW7MBSHEFWDY8TFAMC0JP","body":"Investigating elevated 500 error rate on filtered queries, delete_by_filter operations.","created_at":"2026-05-28T18:52:16Z","display_at":"2026-05-28T17:55:00Z","incident_id":"01KSQYW7MBTXMQDDKCWK8RERZ2","status":"monitoring","updated_at":"2026-05-28T18:55:18Z"}]},{"id":"01KS30V402M7PG5GSP45P9Q703","name":"partial outage in aws-us-west-2","status":"resolved","created_at":"2026-05-20T15:42:34Z","updated_at":"2026-05-20T16:19:38Z","resolved_at":"2026-05-20T16:19:38Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KS32YZWGE2QQKX8V8TG4XPQB","body":"Our deploy has rolled out and affected namespaces have recovered.","created_at":"2026-05-20T16:19:38Z","display_at":"2026-05-20T16:19:38Z","incident_id":"01KS30V402M7PG5GSP45P9Q703","status":"resolved","updated_at":"2026-05-20T16:19:38Z"},{"id":"01KS30V402AAW7GK2TN5X5Y3G9","body":"Our team has identified an issue with a deploy in aws-us-west-2, causing queries and writes to 500 for affected namespaces. We are deploying a fix.","created_at":"2026-05-20T15:42:34Z","display_at":"2026-05-20T15:42:34Z","incident_id":"01KS30V402M7PG5GSP45P9Q703","status":"identified","updated_at":"2026-05-20T16:02:19Z"}]},{"id":"01KPNDFHNHCSRCS892S9QVCZY8","name":"Errors when accessing dashboard","status":"resolved","created_at":"2026-04-20T12:22:41Z","updated_at":"2026-04-20T17:32:27Z","resolved_at":"2026-04-20T17:32:27Z","impact":"critical","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KPNZ6RGDZT00H2E732S2Y4VF","body":"This incident is resolved. Dashboard access is working as usual.\n\nAs noted earlier, this incident only affected the dashboard and had no impact on turbopuffer regions or namespaces!","created_at":"2026-04-20T17:32:27Z","display_at":"2026-04-20T17:32:27Z","incident_id":"01KPNDFHNHCSRCS892S9QVCZY8","status":"resolved","updated_at":"2026-04-20T17:32:27Z"},{"id":"01KPNW3Y6MB485FEZ02F4SHSZS","body":"Traffic has returned to normal. We're monitoring closely!","created_at":"2026-04-20T16:38:29Z","display_at":"2026-04-20T15:00:00Z","incident_id":"01KPNDFHNHCSRCS892S9QVCZY8","status":"monitoring","updated_at":"2026-04-20T16:39:04Z"},{"id":"01KPNKP7S2GKVEP6Q7H7YCDFXK","body":"We're seeing recovery with traffic returning to normal. We're monitoring closely and will share another update shortly.","created_at":"2026-04-20T14:11:11Z","display_at":"2026-04-20T14:11:11Z","incident_id":"01KPNDFHNHCSRCS892S9QVCZY8","status":"monitoring","updated_at":"2026-04-20T14:11:11Z"},{"id":"01KPNHW3XTSV2Y2H5D9ZTD321N","body":"This issue is currently affecting the dashboard and API key management. **turbopuffer regions and namespaces are all operating normally**, and existing API keys continue to authenticate as expected. \n\nThe root cause is a dependency service that we use for metadata. We'll share an update as we make progress.","created_at":"2026-04-20T13:39:27Z","display_at":"2026-04-20T13:39:27Z","incident_id":"01KPNDFHNHCSRCS892S9QVCZY8","status":"identified","updated_at":"2026-04-20T13:42:40Z"},{"id":"01KPNDFHNH29TWN3509EYG12K0","body":"We've confirmed reports of dashboard errors and are actively investigating. Note that the database is not affected - this issue is limited to the dashboard.","created_at":"2026-04-20T12:22:41Z","display_at":"2026-04-20T12:22:41Z","incident_id":"01KPNDFHNHCSRCS892S9QVCZY8","status":"investigating","updated_at":"2026-04-20T12:22:41Z"}]},{"id":"01KN71TQ7SMJYMGMF6T206H4C2","name":"Errors when accessing dashboard","status":"resolved","created_at":"2026-04-02T12:14:00Z","updated_at":"2026-04-02T13:10:34Z","resolved_at":"2026-04-02T13:10:34Z","impact":"critical","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KN7529YFXG1WFPNVCS9T6SHC","body":"We were impacted by [an issue with one of our providers](https://www.vercel-status.com/incidents/5r9bp5y8rql2), which we were able to resolve. We will continue to monitor for regressions.","created_at":"2026-04-02T13:10:34Z","display_at":"2026-04-02T13:10:34Z","incident_id":"01KN71TQ7SMJYMGMF6T206H4C2","status":"resolved","updated_at":"2026-04-02T14:44:19Z"},{"id":"01KN71TQ7SRMNPS5X0ACHE0F7D","body":"We've confirmed reports of dashboard errors and our team is investigating.","created_at":"2026-04-02T12:14:00Z","display_at":"2026-04-02T12:14:00Z","incident_id":"01KN71TQ7SMJYMGMF6T206H4C2","status":"investigating","updated_at":"2026-04-02T12:14:00Z"}]},{"id":"01KMFM12XCJ6YZEZE5H3Z8Y4AT","name":"Elevated 408 and 429s","status":"resolved","created_at":"2026-03-24T09:50:17Z","updated_at":"2026-03-24T10:18:49Z","resolved_at":"2026-03-24T10:18:48Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KMFNNAQRNGKHYAK9HT161WFP","body":"We've observed a complete recovery of traffic.","created_at":"2026-03-24T10:18:49Z","display_at":"2026-03-24T10:18:48Z","incident_id":"01KMFM12XCJ6YZEZE5H3Z8Y4AT","status":"resolved","updated_at":"2026-03-24T10:18:49Z"},{"id":"01KMFN7KNKH4DBYMANQMD9X7J5","body":"Our team has taken remedial action and is observing recovery, we will continue to monitor.","created_at":"2026-03-24T10:11:19Z","display_at":"2026-03-24T10:11:19Z","incident_id":"01KMFM12XCJ6YZEZE5H3Z8Y4AT","status":"monitoring","updated_at":"2026-03-24T10:11:19Z"},{"id":"01KMFM12XCP0HEAVHG1CZCHKBZ","body":"The gcp-us-east4 cluster is experiencing elevated rates of 408 and 429s affecting both reads and writes affecting approximately 4% of traffic. Our team is investigating.","created_at":"2026-03-24T09:50:17Z","display_at":"2026-03-24T09:50:17Z","incident_id":"01KMFM12XCJ6YZEZE5H3Z8Y4AT","status":"investigating","updated_at":"2026-03-24T09:50:17Z"}]},{"id":"01KKXKP72BCH8XYC7FCFVCTVZS","name":"Increased Request Latency and 429s in GCP us-east4","status":"resolved","created_at":"2026-03-17T09:58:01Z","updated_at":"2026-03-17T10:48:08Z","resolved_at":"2026-03-17T10:48:08Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KKXPJ0CVPWR73PP4QQXZ61EG","body":"The region is stable and has fully recovered. We will continue to monitor.","created_at":"2026-03-17T10:48:08Z","display_at":"2026-03-17T10:48:08Z","incident_id":"01KKXKP72BCH8XYC7FCFVCTVZS","status":"resolved","updated_at":"2026-03-17T10:48:08Z"},{"id":"01KKXPBADDF1TMSQDS6AC8NVNM","body":"Latencies have returned to normal and 429s have fully recovered. The issue was related to transient failures in the underlying infrastructure. Our engineers have applied mitigations, and we are continuing to monitor.","created_at":"2026-03-17T10:44:29Z","display_at":"2026-03-17T10:28:00Z","incident_id":"01KKXKP72BCH8XYC7FCFVCTVZS","status":"monitoring","updated_at":"2026-03-26T01:26:19Z"},{"id":"01KKXKP72BS7AN2C2D1Y7JAC29","body":"We are looking into an issue causing increased latency and 429s in us-east4.","created_at":"2026-03-17T09:58:01Z","display_at":"2026-03-17T09:53:00Z","incident_id":"01KKXKP72BCH8XYC7FCFVCTVZS","status":"investigating","updated_at":"2026-03-17T10:53:18Z"}]},{"id":"01KKTXQDTX42CK29FWVZEANA2W","name":"Elevated 500s in region gcp-us-west1","status":"resolved","created_at":"2026-03-16T08:55:43Z","updated_at":"2026-03-16T09:39:58Z","resolved_at":"2026-03-16T09:39:58Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KKV08F11AWX33YA3C27KZ9PX","body":"The region has stabilized and returned to normal operation. We are continuing to monitor!","created_at":"2026-03-16T09:39:58Z","display_at":"2026-03-16T09:39:58Z","incident_id":"01KKTXQDTX42CK29FWVZEANA2W","status":"resolved","updated_at":"2026-03-16T09:39:58Z"},{"id":"01KKTYQ1B67XFFA4MQFD8QN3KM","body":"A fix has been rolled out and 5xx errors in the region have reduced. We are continuing to monitor the situation!","created_at":"2026-03-16T09:12:59Z","display_at":"2026-03-16T09:12:59Z","incident_id":"01KKTXQDTX42CK29FWVZEANA2W","status":"monitoring","updated_at":"2026-03-16T09:14:05Z"},{"id":"01KKTXQDTXSFH0X21Z0PJVT2DC","body":"We are investigating an increase 500's in the region!","created_at":"2026-03-16T08:55:43Z","display_at":"2026-03-16T08:55:43Z","incident_id":"01KKTXQDTX42CK29FWVZEANA2W","status":"investigating","updated_at":"2026-03-16T08:55:43Z"}]},{"id":"01KG3WTDRYBC1W6K7GRHKCGFGQ","name":"Outage in aws-us-west-2","status":"resolved","created_at":"2026-01-29T03:30:15Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2026-01-29T04:47:29Z","impact":"critical","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KG417TVN48H5TVFCYHFT0D4Y","body":"Service has been fully restored. We'll be sharing a full RCA with affected customers as soon as possible.","created_at":"2026-01-29T04:47:29Z","display_at":"2026-01-29T04:47:29Z","incident_id":"01KG3WTDRYBC1W6K7GRHKCGFGQ","status":"resolved","updated_at":"2026-01-29T04:47:29Z"},{"id":"01KG3X5SMK9TTPRGG7YQT992R5","body":"We've identified the problem and rolled out a fix. We're closely monitoring the situation.","created_at":"2026-01-29T03:36:28Z","display_at":"2026-01-29T03:36:28Z","incident_id":"01KG3WTDRYBC1W6K7GRHKCGFGQ","status":"monitoring","updated_at":"2026-01-29T03:36:28Z"},{"id":"01KG3WTDRYWPVJHPK8NWEE03EG","body":"We are investigating reports of an outage in aws-us-west-2.","created_at":"2026-01-29T03:30:15Z","display_at":"2026-01-29T03:30:15Z","incident_id":"01KG3WTDRYBC1W6K7GRHKCGFGQ","status":"investigating","updated_at":"2026-01-29T03:30:15Z"}]},{"id":"01KG2DXE544PC3X9BEJP70BYZQ","name":"Elevated 500s in region gcp-us-east4 due to transient unavailability of turbopuffer query nodes","status":"resolved","created_at":"2026-01-28T13:50:31Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2026-01-28T16:41:29Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KG2QPFXP8FBQ2N4JEFHJFWE8","body":"Service has returned to normal. The cause has been identified and is being addressed.","created_at":"2026-01-28T16:41:29Z","display_at":"2026-01-28T16:41:29Z","incident_id":"01KG2DXE544PC3X9BEJP70BYZQ","status":"resolved","updated_at":"2026-01-28T16:41:29Z"},{"id":"01KG2GHT3RVHYH336VPED1Y95P","body":"We've identified the problem and taken action to stop customer impact.","created_at":"2026-01-28T14:36:35Z","display_at":"2026-01-28T14:36:35Z","incident_id":"01KG2DXE544PC3X9BEJP70BYZQ","status":"monitoring","updated_at":"2026-01-28T14:36:35Z"},{"id":"01KG2DXE54Q5F9YM7WZMGWZY44","body":"Customers may notice a small number of their requests receiving a 500 response due to transient unavailability of turbopuffer query nodes. We have identified a probable cause and are moving to confirming and addressing it.","created_at":"2026-01-28T13:50:31Z","display_at":"2026-01-28T13:50:31Z","incident_id":"01KG2DXE544PC3X9BEJP70BYZQ","status":"investigating","updated_at":"2026-01-28T13:50:31Z"}]},{"id":"01KG21AM747PN6FK3DKHWDS3YH","name":"aws-us-west-2: Increased 429's on a subset of nodes","status":"resolved","created_at":"2026-01-28T10:10:31Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2026-01-28T10:15:00Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KG3570E4VB1KVKJQA06PWJRW","body":"The issue has been resolved. We deployed a fix and confirmed that affected nodes have returned to normal operation.","created_at":"2026-01-28T20:37:41Z","display_at":"2026-01-28T10:15:00Z","incident_id":"01KG21AM747PN6FK3DKHWDS3YH","status":"resolved","updated_at":"2026-01-28T20:39:33Z"},{"id":"01KG21AM7424G7R1QJRKPXSWYX","body":"We identified an issue causing high CPU utilization on a subset of query nodes in us-west-2. Some customers experienced 429 errors and increased query latency during this time.\n\n\nWe have identified the root cause and are deploying a fix. We are actively monitoring the situation and implementing additional safeguards to prevent this from recurring.","created_at":"2026-01-28T10:10:31Z","display_at":"2026-01-28T10:02:00Z","incident_id":"01KG21AM747PN6FK3DKHWDS3YH","status":"monitoring","updated_at":"2026-01-28T20:39:58Z"}]},{"id":"01KG19S6004ZZXC7ZGP4EHTY8J","name":"Degraded performance in aws-us-west-2","status":"resolved","created_at":"2026-01-28T03:19:02Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2026-01-28T04:12:41Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KG1CVDASACVHEFN5HQJX2PMN","body":"Latencies have remained stable.","created_at":"2026-01-28T04:12:41Z","display_at":"2026-01-28T04:12:41Z","incident_id":"01KG19S6004ZZXC7ZGP4EHTY8J","status":"resolved","updated_at":"2026-01-28T04:12:41Z"},{"id":"01KG1AFFD0EGHFW09MCNJR2H56","body":"The region has fully transitioned back to Intel hardware and API response latency has returned to normal. We're monitoring to ensure that latencies remain stable.","created_at":"2026-01-28T03:31:13Z","display_at":"2026-01-28T03:31:13Z","incident_id":"01KG19S6004ZZXC7ZGP4EHTY8J","status":"monitoring","updated_at":"2026-01-28T03:31:13Z"},{"id":"01KG1A8JRCSMM0WWRA52594QS1","body":"We've identified a pathological performance issue when running on ARM hardware. We've shifted the region to run entirely on Intel hardware. ","created_at":"2026-01-28T03:27:27Z","display_at":"2026-01-28T03:09:00Z","incident_id":"01KG19S6004ZZXC7ZGP4EHTY8J","status":"identified","updated_at":"2026-01-28T03:27:45Z"},{"id":"01KG19S6003DQ5TCGANXHJQD1M","body":"We're investigating reports of degraded performance in aws-us-west-2.","created_at":"2026-01-28T03:19:02Z","display_at":"2026-01-28T02:39:00Z","incident_id":"01KG19S6004ZZXC7ZGP4EHTY8J","status":"investigating","updated_at":"2026-01-28T03:22:46Z"}]},{"id":"01KECV1C3Z0GD8HGHZK9DCWFES","name":"indexing backlog in aws-us-west-2","status":"resolved","created_at":"2026-01-07T18:20:52Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2026-01-07T15:00:00Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KECV1C3ZXE064RSQATCKAQMV","body":"incident was resolved","created_at":"2026-01-07T18:20:52Z","display_at":"2026-01-07T15:00:00Z","incident_id":"01KECV1C3Z0GD8HGHZK9DCWFES","status":"resolved","updated_at":"2026-01-07T18:20:52Z"},{"id":"01KECV1C3ZAJ5GTN524M02F61Q","body":"misconfiguration broke indexer auto-scaling, causing a backlog of jobs","created_at":"2026-01-07T18:20:52Z","display_at":"2026-01-07T06:40:00Z","incident_id":"01KECV1C3Z0GD8HGHZK9DCWFES","status":"identified","updated_at":"2026-01-07T18:20:52Z"}]},{"id":"01KCHG3ZFM54ZJCA83J0PA5PRE","name":"Degraded performance in aws-us-west-2","status":"resolved","created_at":"2025-12-15T17:14:51Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-12-15T18:03:54Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KCHJXRTJPJNR9C76M6QTGJ06","body":"The problematic queries have been isolated and traffic has been restored to normal.","created_at":"2025-12-15T18:03:54Z","display_at":"2025-12-15T18:03:54Z","incident_id":"01KCHG3ZFM54ZJCA83J0PA5PRE","status":"resolved","updated_at":"2025-12-15T18:03:54Z"},{"id":"01KCHG3ZFMCXD3N5TQ2ASY1CP4","body":"Some query nodes are currently experiencing excessive load, leading to 429s for certain namespaces. We have identified the problem and have a fix in preparation.","created_at":"2025-12-15T17:14:51Z","display_at":"2025-12-15T17:14:51Z","incident_id":"01KCHG3ZFM54ZJCA83J0PA5PRE","status":"identified","updated_at":"2025-12-15T17:14:51Z"}]},{"id":"01KCGRQ641A6JDXP76539BGGDH","name":"Degraded service (500 responses)","status":"resolved","created_at":"2025-12-15T10:25:55Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-12-15T10:30:58Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KCGS0EA8BSVS4GW49SKQSNG1","body":"Service is stable.","created_at":"2025-12-15T10:30:58Z","display_at":"2025-12-15T10:30:58Z","incident_id":"01KCGRQ641A6JDXP76539BGGDH","status":"resolved","updated_at":"2025-12-15T10:30:58Z"},{"id":"01KCGRQ641C24KBAVXM2P7TQCR","body":"Some of our query nodes restarted repeatedly due to OOMs caused by a code path that was producing very large allocations. We have deployed a fix, and the service has recovered. We are monitoring.","created_at":"2025-12-15T10:25:55Z","display_at":"2025-12-15T10:25:55Z","incident_id":"01KCGRQ641A6JDXP76539BGGDH","status":"monitoring","updated_at":"2025-12-15T10:25:55Z"}]},{"id":"01KCJSVSG6AJ567GBBWS03EG2S","name":"TLS certificate expiration in aws-us-east-1","status":"resolved","created_at":"2025-12-16T05:24:23Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-12-14T21:28:00Z","impact":"critical","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KCJSVSG6BR9V8CNZ9RRR461F","body":"Service has been fully restored.","created_at":"2025-12-16T05:24:23Z","display_at":"2025-12-14T21:28:00Z","incident_id":"01KCJSVSG6AJ567GBBWS03EG2S","status":"resolved","updated_at":"2025-12-16T05:24:23Z"},{"id":"01KCJSVSG6XXMD9K97VYR84FJW","body":"The affected certificates were manually renewed. Connections to aws-us-east-1.turbopuffer.com should once again pass TLS validation.","created_at":"2025-12-16T05:24:23Z","display_at":"2025-12-14T21:25:00Z","incident_id":"01KCJSVSG6AJ567GBBWS03EG2S","status":"monitoring","updated_at":"2025-12-16T05:24:23Z"},{"id":"01KCJSVSG6Z55SMQBF3KA7YZJ2","body":"We've identified that a stuck ACME challenge prevented cert-manager from automatically renewing the TLS certificate for aws-us-east-1.turbopuffer.com.","created_at":"2025-12-16T05:24:23Z","display_at":"2025-12-14T21:23:00Z","incident_id":"01KCJSVSG6AJ567GBBWS03EG2S","status":"identified","updated_at":"2025-12-16T05:24:23Z"},{"id":"01KCJSVSG6RPRBQMS4X9Z74K7R","body":"We are investigating reports of connections to aws-us-east-1.turbopuffer.com failing TLS validation.","created_at":"2025-12-16T05:24:23Z","display_at":"2025-12-14T21:06:00Z","incident_id":"01KCJSVSG6AJ567GBBWS03EG2S","status":"investigating","updated_at":"2025-12-16T05:24:23Z"}]},{"id":"01KC78P4ZDQ4GA11YAGV6SNV2E","name":"degraded performance in aws-us-east-1","status":"resolved","created_at":"2025-12-11T17:52:34Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-12-11T19:31:53Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KC7EC0YRDKFGK17C3ZMGFZ4V","body":"we have not seen any re-occurences of the issue.","created_at":"2025-12-11T19:31:53Z","display_at":"2025-12-11T19:31:53Z","incident_id":"01KC78P4ZDQ4GA11YAGV6SNV2E","status":"resolved","updated_at":"2025-12-11T19:31:53Z"},{"id":"01KC7D208GMPE2VKSA5751WS99","body":"we've removed problematic query nodes from rotation and are monitoring for further issues.","created_at":"2025-12-11T19:08:57Z","display_at":"2025-12-11T19:08:56Z","incident_id":"01KC78P4ZDQ4GA11YAGV6SNV2E","status":"monitoring","updated_at":"2025-12-11T19:08:57Z"},{"id":"01KC78P4ZDV58R0WBZP2V5CZ60","body":"some namespaces in aws-us-east-1 may experience degraded performance and request timeouts. our team is working to rectify the issue.","created_at":"2025-12-11T17:52:34Z","display_at":"2025-12-11T17:52:34Z","incident_id":"01KC78P4ZDQ4GA11YAGV6SNV2E","status":"investigating","updated_at":"2025-12-11T17:52:34Z"}]},{"id":"01KAV0K9DX14C0DMTH7C3RF5BV","name":"Elevated 500s in gcp-us-west1","status":"resolved","created_at":"2025-11-24T13:24:36Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-11-24T13:27:41Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01KAV0RXJ72XKF81C973CQ4ZV1","body":"We have taken steps to remediate the 500s, which have now subsided. We're investigating the root cause.","created_at":"2025-11-24T13:27:41Z","display_at":"2025-11-24T13:27:41Z","incident_id":"01KAV0K9DX14C0DMTH7C3RF5BV","status":"resolved","updated_at":"2025-11-24T13:27:41Z"},{"id":"01KAV0K9DXKDMD9ZVYHHQ57NE4","body":"We're investigating an increase in 500 responses in the region.","created_at":"2025-11-24T13:24:36Z","display_at":"2025-11-24T13:24:36Z","incident_id":"01KAV0K9DX14C0DMTH7C3RF5BV","status":"identified","updated_at":"2025-11-24T13:24:36Z"}]},{"id":"01K81J9ST40XSW58BVBDX94D87","name":"Partial dashboard outage","status":"resolved","created_at":"2025-10-20T19:42:21Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-10-21T02:31:00Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01K83X4T8M8113FEM4B8MSSVM0","body":"The upstream incident is resolved.","created_at":"2025-10-21T17:30:21Z","display_at":"2025-10-21T02:31:00Z","incident_id":"01K81J9ST40XSW58BVBDX94D87","status":"resolved","updated_at":"2025-10-21T17:30:48Z"},{"id":"01K81S8S5A185JTBX9VAKRFQT9","body":"Dashboard error rates have subsided. We continue to monitor the situation while we wait for confirmation from our authentication provider that the incident has resolved.","created_at":"2025-10-20T21:44:08Z","display_at":"2025-10-20T19:52:00Z","incident_id":"01K81J9ST40XSW58BVBDX94D87","status":"monitoring","updated_at":"2025-10-20T21:47:22Z"},{"id":"01K81J9ST4FGPDCAX0DS3HVEB1","body":"Some users are reporting issues accessing the turbopuffer dashboard due to a partial outage with our authentication provider: \u003chttps://status.workos.com/incidents/rdz189wnjhyn\u003e\n\nturbopuffer APIs are unaffected.","created_at":"2025-10-20T19:42:21Z","display_at":"2025-10-20T18:32:00Z","incident_id":"01K81J9ST40XSW58BVBDX94D87","status":"identified","updated_at":"2025-10-20T21:47:15Z"}]},{"id":"01K80A1TPH9P1KJ3Z28NCDQ274","name":"Outage in aws us-east-1","status":"resolved","created_at":"2025-10-20T07:58:57Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-10-20T23:18:06Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01K81YMVQ3BB80K25B5ADK3664","body":"AWS has reported that the upstream incident is resolved.","created_at":"2025-10-20T23:18:06Z","display_at":"2025-10-20T23:18:06Z","incident_id":"01K80A1TPH9P1KJ3Z28NCDQ274","status":"resolved","updated_at":"2025-10-20T23:18:06Z"},{"id":"01K81S047FJF67ESJA5K4MSZAG","body":"The turbopuffer service remains healthy in the aws-us-east-1 region. We continue to closely monitor the situation while AWS continues to report service disruption in the region.","created_at":"2025-10-20T21:39:24Z","display_at":"2025-10-20T21:39:24Z","incident_id":"01K80A1TPH9P1KJ3Z28NCDQ274","status":"monitoring","updated_at":"2025-10-20T21:39:24Z"},{"id":"01K810HQJ98G65KEZP588GZ8AB","body":"AWS is experiencing a major disruption in region us-east-1 impacting multiple services (\u003chttps://health.aws.amazon.com/health/status?eventID=arn:aws:health:us-east-1::event/MULTIPLE_SERVICES/AWS_MULTIPLE_SERVICES_OPERATIONAL_ISSUE/AWS_MULTIPLE_SERVICES_OPERATIONAL_ISSUE_BA540_514A652BE1A\u003e). This caused turbopuffer to become unavailable in that region starting at 7:50 UTC. We've rolled out mitigations which resulted in a full recovery of turbopuffer at 8:20 UTC (despite the ongoing AWS outage). An existing mitigation to guard against EC2 instance launch issues has prevented the long-running EC2 degradation from further affecting turbopuffer availability. The AWS incident is ongoing. We will continue monitoring the cluster's health.","created_at":"2025-10-20T14:32:07Z","display_at":"2025-10-20T14:32:07Z","incident_id":"01K80A1TPH9P1KJ3Z28NCDQ274","status":"monitoring","updated_at":"2025-10-20T14:32:07Z"},{"id":"01K80CAJV5HMNRM8T7TCYHP6XG","body":"We have deployed mitigations and are monitoring the situation. ","created_at":"2025-10-20T08:38:41Z","display_at":"2025-10-20T08:38:41Z","incident_id":"01K80A1TPH9P1KJ3Z28NCDQ274","status":"monitoring","updated_at":"2025-10-20T08:38:41Z"},{"id":"01K80A1TPHFTGQ7VQAVV6RE8C5","body":"[AWS us-east-1 is experiencing a major outage and many services are affected. ](https://health.aws.amazon.com/health/status)\n\nturbopuffer clients in aws-us-east-1 will see elevated error rate and latencies during this outage.\n\nWe're looking at mitigating the impact.","created_at":"2025-10-20T07:58:57Z","display_at":"2025-10-20T07:58:57Z","incident_id":"01K80A1TPH9P1KJ3Z28NCDQ274","status":"investigating","updated_at":"2025-10-20T08:01:05Z"}]},{"id":"01K80GPKHX8B73SMG6QPC4AYYP","name":"Degraded performance in gcp-us-east4","status":"resolved","created_at":"2025-10-20T09:55:09Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-10-20T17:22:02Z","impact":"minor","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01K81A8VZ3MYF2WWZHK0CDYHV2","body":"Service is stable. We are continuing to investigate the root cause with high priority.","created_at":"2025-10-20T17:22:02Z","display_at":"2025-10-20T17:22:02Z","incident_id":"01K80GPKHX8B73SMG6QPC4AYYP","status":"resolved","updated_at":"2025-10-20T17:22:02Z"},{"id":"01K80HQ2YBKV2MAT7BBWAPCT9G","body":"We have observed an increase in timeouts and a slow down in indexing starting at 9:38 UTC. The service has returned to normal at 10:00 UTC. We are monitoring and investigating the root cause.","created_at":"2025-10-20T10:12:53Z","display_at":"2025-10-20T10:12:53Z","incident_id":"01K80GPKHX8B73SMG6QPC4AYYP","status":"monitoring","updated_at":"2025-10-20T10:12:53Z"},{"id":"01K80GPKHXRZAMVDS2F7ZDQFXS","body":"we're currently observing degraded performance in gcp-us-east4 leading to elevated rates of 408s and 429s. Our team is currently investigating.","created_at":"2025-10-20T09:55:09Z","display_at":"2025-10-20T09:55:09Z","incident_id":"01K80GPKHX8B73SMG6QPC4AYYP","status":"investigating","updated_at":"2025-10-20T09:55:09Z"}]},{"id":"01K7P4NKAZNH02NK6T055BFND6","name":"Partial outage in gcp-us-east4","status":"resolved","created_at":"2025-10-16T09:12:29Z","updated_at":"2026-03-12T18:21:09Z","resolved_at":"2025-10-16T10:52:20Z","impact":"major","page_id":"01K0Q1X4P7X87YAH6FHH452BES","incident_updates":[{"id":"01K7PNXFGDSQ8ERQZ251Z0SQV4","body":"Full recovery as of 12:24 PM UTC","created_at":"2025-10-16T14:13:53Z","display_at":"2025-10-16T14:13:53Z","incident_id":"01K7P4NKAZNH02NK6T055BFND6","status":"resolved","updated_at":"2025-10-16T14:13:53Z"},{"id":"01K7PC4936407J35WJDM1SVSAB","body":"We are seeing degraded performance again as latency to Google Cloud Storage. We are continuing to monitor and work with the GCP team.","created_at":"2025-10-16T11:22:50Z","display_at":"2025-10-16T11:05:00Z","incident_id":"01K7P4NKAZNH02NK6T055BFND6","status":"monitoring","updated_at":"2025-10-16T11:23:25Z"},{"id":"01K7PACEAFX1WZR705EHB90JWP","body":"Full recovery as of 10:44am UTC.\n\nWe are investigating the root cause with the GCP team.","created_at":"2025-10-16T10:52:20Z","display_at":"2025-10-16T10:52:20Z","incident_id":"01K7P4NKAZNH02NK6T055BFND6","status":"resolved","updated_at":"2025-10-16T10:52:50Z"},{"id":"01K7PA36ANCP2ZPWB7K4527PMD","body":"Substantial improvement starting 10:35am UTC. We are continuing to investigate and work directly with GCP to resolve this fully.","created_at":"2025-10-16T10:47:17Z","display_at":"2025-10-16T10:47:17Z","incident_id":"01K7P4NKAZNH02NK6T055BFND6","status":"monitoring","updated_at":"2025-10-16T10:52:28Z"},{"id":"01K7P4NKAZNPTQA9W2TCGWDSJ1","body":"8:30am UTC - 9:25am UTC: Degraded performance on queries and writes in gcp-us-east4 from increased latency to Google Cloud Storage.\n\n9:25am UTC: Escalated to partial outage both write and read paths due to climbing latency and timeouts from Google Cloud Storage.\n\n* Queries may see elevated rates of 429s, and 500s.\n* Writes may observe elevated rates of 408s, and 500s.\n\nOur team is working to restore full system performance, and we are working with GCP to restore service as soon as possible. ","created_at":"2025-10-16T09:12:29Z","display_at":"2025-10-16T08:30:00Z","incident_id":"01K7P4NKAZNH02NK6T055BFND6","status":"identified","updated_at":"2025-10-16T11:07:26Z"}]}]}