{
"vendor": "Circle CI",
"slug": "circle-ci",
"platform": "statuspage",
"status_url": "https://status.circleci.com",
"last_checked": "2026-09-16T12:28:20Z",
"last_state": "ok",
"history_backfilled": true,
"first_watched": "2026-09-04T07:06:16Z",
"incidents": [
{
"body": "The incident impacting pipeline data in the UI has now been resolved.  We thank you for your patience while our engineers worked on reducing the delays.",
"first_seen": "2026-09-16T12:28:20Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-09-15T17:02:45.189Z",
"resolved_inferred": false,
"started_at": "2026-09-15T14:42:25.100Z",
"state": "resolved",
"title": "Customers may experience delays in UI updates",
"updated_at": "2026-09-15T17:02:45.209Z",
"url": "https://stspg.io/0djh5p3ll96t"
},
{
"body": "Wait times for macOS jobs are back to normal.",
"first_seen": "2026-09-14T12:29:13Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-09-14T12:59:40.333Z",
"resolved_inferred": false,
"started_at": "2026-09-14T11:59:15.782Z",
"state": "resolved",
"title": "Increased wait times for macOS jobs",
"updated_at": "2026-09-14T12:59:40.354Z",
"url": "https://stspg.io/d885vgdcmp7p"
},
{
"body": "This incident has been resolved.  A small number of customers who created pipelines between 18:53 UTC and 19:24 UTC that contained config using deprecated config syntax will have seen their build-type jobs fail to start. Customers whose jobs or pipelines failed may rerun them.",
"first_seen": "2026-09-11T12:29:42Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-09-10T20:26:50.000Z",
"resolved_inferred": false,
"started_at": "2026-09-10T20:25:39.781Z",
"state": "resolved",
"title": "Issues loading pipelines for a small number of projects",
"updated_at": "2026-09-10T22:39:19.147Z",
"url": "https://stspg.io/l0tsrtv1tkbl"
},
{
"body": "We've seen no further impact to queue times and everything has returned to normal. We appreciate your patience while we worked to resolve and monitor this issue.",
"first_seen": "2026-09-04T15:18:55Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-09-04T16:55:41.439Z",
"resolved_inferred": false,
"started_at": "2026-09-04T15:16:02.197Z",
"state": "resolved",
"title": "macOS Job Delays",
"updated_at": "2026-09-04T16:55:41.455Z",
"url": "https://stspg.io/l8rtj1qhr625"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-26T16:10:23.938Z",
"resolved_inferred": false,
"started_at": "2026-08-26T15:43:38.082Z",
"state": "resolved",
"title": "Intermittent errors when viewing plan usage UIs or calling plan usage APIs",
"updated_at": "2026-08-26T16:10:23.956Z",
"url": "https://stspg.io/961zy5b9fnjk"
},
{
"body": "Between 19:49 UTC and 21:07 UTC on August 25, 2026, customers attempting to log in to CircleCI using GitHub were unable to sign in, and received an error from GitHub stating the callback URL was invalid. Customers who were already logged in were not affected. The issue has been resolved and GitHub login is functioning normally. We thank you for your patience while our team worked on implementing a fix.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-25T20:00:00.000Z",
"resolved_inferred": false,
"started_at": "2026-08-25T20:00:00.000Z",
"state": "resolved",
"title": "GitHub Login Disruption",
"updated_at": "2026-08-25T23:21:18.282Z",
"url": "https://stspg.io/ns4jz57ynx52"
},
{
"body": "Insights data is good once more. Thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-17T21:37:03.148Z",
"resolved_inferred": false,
"started_at": "2026-08-17T21:06:09.742Z",
"state": "resolved",
"title": "Insight data is currently delayed",
"updated_at": "2026-08-17T21:37:03.169Z",
"url": "https://stspg.io/gwc69d86r5sy"
},
{
"body": "GitHub's APIs appear to be operating normally.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "major",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-17T19:05:30.477Z",
"resolved_inferred": false,
"started_at": "2026-08-17T14:04:21.229Z",
"state": "resolved",
"title": "GitHub Incidents impacting CircleCI functionality",
"updated_at": "2026-08-17T19:05:30.501Z",
"url": "https://stspg.io/mk4lvmlx939f"
},
{
"body": "Following planned maintenance that ended at 13:00 UTC on August 15, some impact to job processing continued for approximately 30 minutes beyond the window we announced.\n\nSome customers continued to see delays in jobs starting, along with a small number of jobs failing with infrastructure errors, until approximately 13:30 UTC.\n\nThis has been resolved and job processing has returned to normal. Customers whose jobs failed during this window may rerun affected jobs. We thank you for your patience while our team worked on implementing a fix and apologize for any inconvenience the extended impact may have caused.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-15T17:00:00.000Z",
"resolved_inferred": false,
"started_at": "2026-08-15T13:58:54.814Z",
"state": "resolved",
"title": "Delays starting jobs following planned maintenance",
"updated_at": "2026-08-15T13:58:54.874Z",
"url": "https://stspg.io/v83b8dvfq3hq"
},
{
"body": "The issue impacting customers who use GitHub as their VCS while GitHub was undergoing a service degradation (https://www.githubstatus.com/incidents/76t89hbfb09h) has now been resolved.  GitHub has resolved the underlying issue and the affected functionality has returned to normal. \n\nWe thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-12T16:43:37.854Z",
"resolved_inferred": false,
"started_at": "2026-08-12T16:33:41.163Z",
"state": "resolved",
"title": "GitHub service degradation may impact customers using GitHub",
"updated_at": "2026-08-12T16:43:37.870Z",
"url": "https://stspg.io/5n5fpq53kkhh"
},
{
"body": "Between 13:50 UTC and approximately 17:30 UTC on August 11, 2026, some customers experienced delays of up to an hour in pipeline status updates and in notification delivery. \n\nBetween 16:40 UTC and 17:00 UTC within that same window, a small number of customers also saw newly created pipelines fail to process, appearing in an errored state in the UI and API. \n\nThe issue has been resolved and all affected functionality has returned to normal. Customers whose pipelines errored during that window can retrigger them. \n\nWe thank you for your patience while our team worked on implementing a fix.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-11T18:00:00.000Z",
"resolved_inferred": false,
"started_at": "2026-08-11T17:42:47.509Z",
"state": "resolved",
"title": "Delayed pipeline updates and pipeline processing failures",
"updated_at": "2026-08-11T17:42:47.576Z",
"url": "https://stspg.io/b811tvs48x2k"
},
{
"body": "The Usage API issue has been resolved. Data has been loaded for yesterday, 8/5/2026. We appreciate your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-06T19:54:25.375Z",
"resolved_inferred": false,
"started_at": "2026-08-06T13:06:17.519Z",
"state": "resolved",
"title": "Usage API data delayed for 8/5",
"updated_at": "2026-08-06T19:54:25.388Z",
"url": "https://stspg.io/kntq9cf4jgpk"
},
{
"body": "The upstream issue has been resolved, and the insights data from the last 24 hours has caught up. Thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-05T21:01:12.564Z",
"resolved_inferred": false,
"started_at": "2026-08-05T16:58:27.652Z",
"state": "resolved",
"title": "Insights service data is lagging",
"updated_at": "2026-08-05T21:01:12.581Z",
"url": "https://stspg.io/6qdzqdj5ww81"
},
{
"body": "Queue times for Android and Windows jobs have reduced significantly over the last hour. You may still notice brief, isolated queuing at times, but this is no longer at incident-level impact. This was related to capacity constraints with a third-party infrastructure provider. We thank you for your patience while we monitored the situation. If you have any issues, please reach out to our Support team.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-08-04T23:40:23.795Z",
"resolved_inferred": false,
"started_at": "2026-08-04T19:27:28.000Z",
"state": "resolved",
"title": "Increased Job Queue Times: Windows, Android, and GPU",
"updated_at": "2026-08-04T23:40:23.814Z",
"url": "https://stspg.io/525771jb0gj4"
},
{
"body": "The issue impacting customers who use GitHub as their VCS while GitHub was undergoing a service degradation (https://www.githubstatus.com/incidents/yjysg0xrl67m) has now been resolved.  GitHub has resolved the underlying issue and the affected functionality has returned to normal. \n\nWe thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-24T17:41:31.000Z",
"resolved_inferred": false,
"started_at": "2026-07-24T16:35:44.272Z",
"state": "resolved",
"title": "GitHub service degradation may impact customers using GitHub",
"updated_at": "2026-07-24T17:41:46.393Z",
"url": "https://stspg.io/8pvsx88khry9"
},
{
"body": "Between 19:24 UTC on 22 July and 01:50 UTC on 23 July, customers using the gen3 preview machine (Linux VM) resource classes experienced delays and, in some cases, jobs failing to start. We have removed the gen3 resource classes while we address stability issues affecting them. Unfortunately, workflows that we were unable to provision compute capacity for during this period were cancelled. Affected customers should change their resource class from gen3 to gen2 and rerun those workflows. \n\nWe thank you for your patience while our team worked to resolve this.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-23T02:05:30.377Z",
"resolved_inferred": false,
"started_at": "2026-07-22T19:57:15.421Z",
"state": "resolved",
"title": "Delays in starting  jobs using the gen3 resource class",
"updated_at": "2026-07-23T02:05:30.391Z",
"url": "https://stspg.io/jjsrf04jn924"
},
{
"body": "Between 03:18 UTC and 06:04 UTC on July 22, 2026, a subset of customers were unable to access app.circleci.com. The issue has been resolved and access has returned to normal. \n\nWe thank you for your patience while our team worked on implementing a fix.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-22T06:27:34.150Z",
"resolved_inferred": false,
"started_at": "2026-07-22T06:13:48.941Z",
"state": "resolved",
"title": "Errors loading app.circleci.com",
"updated_at": "2026-07-22T06:27:34.167Z",
"url": "https://stspg.io/cw24pb57b6s9"
},
{
"body": "## Summary\n\nOn July 20, 2026 from 00:22 to 01:45 UTC, CircleCI customers using our GitHub integrations experienced failures running pipelines and experienced workflows getting stuck. During this incident, some pipelines triggered by users using GitHub failed to start or ran as error pipelines. Customers whose pipelines failed or ran as error pipelines during this window should re-run them.\n\nThis was caused by an [upstream API degradation at GitHub](https://www.githubstatus.com/incidents/ph5nns5y4gxj), which impacted many GitHub APIs that CircleCI uses to trigger and run pipelines.\n\nBy 01:45 UTC, GitHub APIs recovered, and customer pipelines ran normally.\n\nThe original CircleCI status page can be found [here](https://status.circleci.com/incidents/9lvbbbs9l87b).\u00a0\n\n## What Happened\n\n\\(all times UTC\\)\n\nBeginning\u00a000:22\u00a0on July 20, 2026, GitHub began returning elevated errors for the following API requests:\n\n* `applications/*/token`\n* `repos/*/commits`\n* `repos/*/*/contents/*`\n* `repos/*/*/hooks`\n* `repos/*/*/hooks/*`\n* `repos/*/*/keys`\n* `repos/*/*/pulls`\n* `repos/*/*/statuses/*`\n\nCircleCI relies on these requests to correctly trigger and run pipelines.\n\nAt 00:22, our internal monitoring alerted us to the problem. Our team began investigating, and found that a small number of pipelines belonging to GitHub projects either failed to start or ran as error pipelines. This included scheduled pipelines and workflows. In addition, a small number of customer workflows experienced stuck jobs and required a re-run of the workflow to fix.\n\nAt\u00a001:45, GitHub recovered and pipeline processing returned to normal operating levels. Only pipelines triggered during the incident window were affected, and customers should re-run those pipelines.\n\n## Future Prevention and Process Improvement\n\nWe are actively working to improve the resilience of pipeline processing during GitHub service disruptions to reduce the customer impact of similar incidents in the future. We are scoping features that will help customers recover with reduced manual steps after incidents resolve.\n\nCustomer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. Please reach out to our support team with any questions or concerns.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-20T01:45:35.921Z",
"resolved_inferred": false,
"started_at": "2026-07-20T00:35:59.020Z",
"state": "postmortem",
"title": "Errors with GitHub APIs delaying workflows",
"updated_at": "2026-07-23T21:10:52.591Z",
"url": "https://stspg.io/96ffdz58cxdr"
},
{
"body": "An incident affecting GitHub's API caused CircleCI customers to experience errors loading the web app and signing in, along with pipelines and scheduled workflows failing to start or running late. GitHub has resolved the underlying incident (https://www.githubstatus.com/) and all affected functionality has returned to normal.\n\nCustomers whose jobs or pipelines failed may rerun them. Scheduled pipelines and workflows that were missed during the incident will not run automatically and will need to be re-triggered.\n\nWe thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-17T00:26:36.776Z",
"resolved_inferred": false,
"started_at": "2026-07-16T23:04:18.952Z",
"state": "resolved",
"title": "Errors accessing CircleCI and delays starting pipelines and scheduled workflows",
"updated_at": "2026-07-17T00:26:36.789Z",
"url": "https://stspg.io/jftpxjjgvqxy"
},
{
"body": "On July 13 between 15:30 UTC and 22:40 UTC, we experienced delays starting jobs on Docker (Gen 2) resource classes due to capacity constraints from our cloud provider. The same issue recurred on July 14 between 15:45 UTC and 22:50 UTC.\n\nDelays reached up to 25 minutes on July 13 and up to 24 minutes on July 14. The medium+ and 2 X-large+ resource classes saw the longest waits. Earlier updates in this incident reported figures based on average wait times, which didn't reflect the maximum impact some customers may have experienced.\n\nWait times have now returned to normal. We are continuing to work with our cloud provider to add capacity ahead of the next peak period. Thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-15T00:32:55.545Z",
"resolved_inferred": false,
"started_at": "2026-07-14T22:54:42.726Z",
"state": "resolved",
"title": "Delays starting jobs on Docker (Gen 2) resource classes",
"updated_at": "2026-07-15T00:32:55.560Z",
"url": "https://stspg.io/0t40l14n15yl"
},
{
"body": "## Summary\n\nFrom 01:33 UTC to 05:18 UTC on July 14, 2026, customers with a Bitbucket-linked identity were unable to log in to CircleCI, and some customers experienced workflows that failed to start or update, due to a change in how Bitbucket's OAuth service reports account permissions. At 05:07 UTC, we deployed a fix that corrected how our systems read the updated permissions field. Some customers needed to log out and re-authenticate their Bitbucket integration after we deployed the fix. Our systems continued processing the backlog of affected jobs until 08:39 UTC.\n\nWe thank our customers for their patience as we resolved this incident. Please reach out to our support team with any questions or concerns.\n\nThe status page for this incident can be found [here](https://status.circleci.com/incidents/gsyjwybg477g).\n\n## Background\n\nCircleCI supports logging in with a GitHub account, a Bitbucket account, or an email and password. When a customer logs in with a Bitbucket-linked identity, or when CircleCI needs to refresh the customer's access on their behalf, we exchange an authorization token with Bitbucket's OAuth service. This exchange includes a list of the permissions, or \"scopes,\" the customer has granted us, which Bitbucket and CircleCI use to confirm what CircleCI is authorized to do on the customer\u2019s behalf.\n\n## What Happened\n\n\\(All times UTC\\)\n\nOn April 8, 2026, [Bitbucket announced a change to its OAuth service](https://developer.atlassian.com/cloud/bitbucket/changelog/#CHANGE-3139): it would be renaming the field used to report a customer's granted permissions. Bitbucket ran a transition period during which both the old and new field names were available, then fully removed the old field name on May 4, 2026. Our systems had not been updated to recognize the new field name, so once Bitbucket fully phased out the old one, requests that depended on it began to fail.\n\nAt 01:33 on July 14, 2026, our systems began failing to process permissions information returned for Bitbucket-linked accounts, because our systems still expected the old permissions structure. This caused login attempts for all Bitbucket-linked accounts to fail, including for customers who log in with GitHub but have a Bitbucket identity linked to their account.\n\nAt 01:59, automated monitoring alerted our engineering team to a spike in errors on the affected systems. The team began investigating immediately, confirmed customer impact at 02:38, and alerted customers via our status page at 02:51. By 03:00, the team had isolated the failures to the renamed Bitbucket field.\n\nBeginning at 03:17, workflow status updates for Bitbucket pipelines belonging to customers with an expired access token began to be dropped. The Bitbucket permissions check is used broadly across our platform, and these permissions failures also affected some of our internal job-processing systems. This caused a subset of workflows to become stuck without a final status, and caused some pull requests to show missing or stuck status checks.\n\nAt 04:21, the team deployed an initial fix that resolved the underlying issue, restoring the login flow and returning our internal permissions system to normal operation. Some affected customers needed to log out and log back in to pick up the fix. By 05:07, the team deployed two additional fixes so our systems would begin accepting Bitbucket's new permissions field format going forward. We resolved the incident at 05:18.\n\nSome customers did need to log out and re-authenticate their Bitbucket integration before their account fully recovered. Our systems continued processing a backlog of affected jobs, returning to normal levels by approximately 08:39.\n\n## Future Prevention and Process Improvement\n\nWe are taking the following steps to prevent a recurrence and improve our response time:\n\n**We are hardening our authorization code against upstream API changes.** This incident happened because our system did not gracefully handle a renamed field in a response from a third-party identity provider. We will be immediately updating our authorization code so that unexpected or missing fields from GitHub, Bitbucket, and GitLab are handled safely.\n\n**We are improving how we track upstream provider changes.** We currently already monitor changelogs from our identity providers for exactly this kind of breaking change, but we didn't add this particular provider change to our monitoring in time to catch it before it shipped. We are auditing and expanding this monitoring so provider announcements reach our team before they affect customers.\n\n**We are improving how we classify and communicate incidents in their earliest minutes.** This incident's initial classification did not immediately reflect its customer-facing severity. We are refining our incident tooling and guidance to help engineers identify and communicate customer impact more quickly.\n\n**We are improving the resilience of our workflow-processing pipeline.** A downstream failure in this incident caused some workflow status updates to be dropped rather than retried or clearly surfaced. We are reviewing this system's retry and error-handling behavior so similar downstream failures are more visible and easier to recover from.\n\nCustomer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. Please reach out to our support team with any questions or concerns.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-14T05:15:18.525Z",
"resolved_inferred": false,
"started_at": "2026-07-14T02:49:37.834Z",
"state": "postmortem",
"title": "Login issues for some Bitbucket users",
"updated_at": "2026-07-14T22:31:39.552Z",
"url": "https://stspg.io/lzl0fmm6xmts"
},
{
"body": "## Summary\n\nFrom 18:00 UTC on July 13, 2026 to 23:18 UTC on July 14, 2026, some customers running jobs on CircleCI's macOS fleet experienced jobs that hung or failed to progress during GitHub fetch steps. For these customers, GitHub fetch steps which would ordinarily take 1-2 minutes took 20-30 minutes, and timed out. The issue was caused by a capacity and routing problem on the network path between our Mac infrastructure provider and GitHub, and the associated intermittent networking issues slowed down fetches from GitHub. During the entire incident, our team worked directly with our Mac infrastructure provider to identify and reroute traffic away from the affected network paths. We resolved an initial occurrence at 19:40 UTC on July 13. The issue recurred at 21:52 UTC that same day, we reopened the incident, and it was fully resolved by 23:18 UTC on July 14.\n\nOur macOS fleet had a similar issue several weeks ago. From 20:18 UTC on June 24, 2026 to 03:09 UTC on June 25, 2026, customers experienced a [similar incident](https://status.circleci.com/incidents/gvysjmkf4ct9) affecting the ability of macOS jobs to reach GitHub. That incident was also caused by a network routing problem in our Mac infrastructure provider.\n\nWe thank our customers for their patience while we worked through this incident. Please see below for specific actions which CircleCI and our Mac infrastructure provider will be taking. Please reach out to our support team with any questions or concerns.\n\nThe status pages for this incident can be found [here](https://status.circleci.com/incidents/n1tc2lw9q7l0) and [here](https://status.circleci.com/incidents/7cnl777wp9qz).\n\n## Background\n\nCircleCI's macOS jobs are hosted on a third-party Mac infrastructure provider. That provider connects to the broader internet, including services like GitHub, over multiple redundant upstream network paths. When one of those paths is experiencing reduced capacity or a routing problem, jobs that are fetching code or dependencies from a destination along that network path can slow down or hang intermittently, even when CircleCI's own platform, the third-party infrastructure provider, and the destination service are all operating normally.\n\n## What Happened\n\n_\\(All times UTC\\)_\n\nAt 18:00 on July 13, some customers running macOS jobs began experiencing intermittent failures and delays fetching code and dependencies from GitHub. We opened an investigation and alerted our customers via our status page at 18:55. By 19:04, our Mac infrastructure provider identified increased latency on one of its network paths and rerouted traffic around it. Fetch times recovered, and we moved the incident to `Monitoring` at 19:29, and to `Resolved` at 19:40.\n\nAt 20:44 UTC, customers reported that the issue had returned. We reopened the incident and updated our status page to `Investigating` at 21:52. Over the following two hours, we worked closely with our infrastructure provider to gather diagnostic data in an effort to isolate the affected network path. In the meantime, we published guidance recommending that customers increase their `no_output_timeout` setting to 15\u201320 minutes, to allow more time for GitHub fetches to complete during the intermittent slowdowns, and moved the status page to `Monitoring` at 23:35.\n\nAfter publishing the workaround guidance at 23:35 UTC on July 13, we expected the underlying network issue to improve overnight. It did not. At 13:59 UTC on July 14, we confirmed that customers were still experiencing intermittent failures and moved the status page back to `Investigating`. Our engineering team reproduced the failure directly using our own testing tools, which helped confirm this was a general `git-fetch` issue rather than something specific to any particular build tool, container image, or software update. At 15:35, we updated the status page to share that our infrastructure provider was testing changes to its network configuration to isolate the source of the issue. At 16:31, we updated the status page again to share that we'd identified the cause as reduced inbound bandwidth on a specific route, and that we were working with our provider to shift traffic to a different route. At 16:36, our infrastructure provider identified the affected network paths and shifted traffic away from them, fetch times recovered, and at 16:58, we updated the status page to `Monitoring`.\n\nBy early afternoon, some customers were again seeing slow fetch times, and we moved the status page back to `Investigating` at 20:18. Our infrastructure provider identified two additional network paths with degraded performance and shifted traffic away from them by 22:07. Fetch times stabilized over the following hour, and we marked the incident `Resolved` at 23:18.\n\nIn total, customers running Mac jobs may have experienced intermittent job slowdowns or failures for portions of the window between 18:00 UTC on July 13 and 23:18 UTC on July 14. Our infrastructure provider has since reported clean test results across its network and believes the underlying capacity issue originated further upstream, closer to GitHub, rather than within its own network. We are continuing to monitor this closely.\n\n## Future Prevention and Process Improvement\n\nWe are taking the following steps to prevent a recurrence and improve our response time:\n\n**We are building additional automated monitoring for macOS fleet GitHub-fetch performance.** Our extensive automated monitoring did not catch this incident. We are now adding end-to-end monitoring for GitHub fetches in particular, in order to detect similar issues proactively.\n\n**We are working directly with our Mac infrastructure provider on faster, more proactive detection.** We are asking our provider to build monitoring that can detect a degraded network path and reroute around it automatically, rather than relying on CircleCI to identify and request a reroute during an active incident.\n\n**We are turning the diagnostic tooling we built during this incident into a standing capability.** This will let us detect and reproduce this class of network failure on demand going forward, rather than assembling test infrastructure during an active incident.\n\n**We are reviewing our incident response process for a sufficient confirmation window before marking a network-related incident Resolved.** This incident briefly recurred after an early, premature resolution; we are formalizing a minimum period of confirmed-clean monitoring before closing incidents of this type. We will continue to provide regular updates along the way as investigation and remediation continues.\n\n**We are actively moving from a single provider for Mac infrastructure** **to multiple providers.** This will allow CircleCI to route incoming customer workloads to the most available and best-performing provider.\n\nCustomer experience is our top priority, and we commit to continually improving the reliability of our systems to match the trust that our customers place in us. Please reach out to our support team with any questions or concerns.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-14T23:18:34.925Z",
"resolved_inferred": false,
"started_at": "2026-07-13T21:52:02.888Z",
"state": "postmortem",
"title": "Issues with network routing for Mac jobs",
"updated_at": "2026-07-16T01:56:42.744Z",
"url": "https://stspg.io/vg35mbl8lxpg"
},
{
"body": "We've confirmed the fix with our infrastructure provider and successful tests on our end.  If customers continue to see any failed runs, please run again for a successful connection and reach out to Support if any further issues.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-13T19:40:22.542Z",
"resolved_inferred": false,
"started_at": "2026-07-13T18:55:54.301Z",
"state": "resolved",
"title": "Issues with network routing for Mac jobs",
"updated_at": "2026-07-13T19:40:22.558Z",
"url": "https://stspg.io/vc3sdl5vsqyr"
},
{
"body": "## Summary\n\nOn July 2, 2026, from 15:12 UTC to 16:44 UTC, some CircleCI customers experienced failures starting pipelines and difficulty accessing the CircleCI UI, including an inability to log in. Customers triggering pipelines via the API may also have had pipelines fail to run.\n\nThe incident originated from a combination of internal maintenance and customer-initiated project deletions running concurrently, which caused elevated load on a data service. The connection layer between our API and that data service was not configured to time out slow responses under the elevated load conditions seen during this incident. This eventually exhausted available threads and prevented the request routing layer from handling any other requests, resulting in customer-facing errors.\n\nThe affected services were scaled up and the internal maintenance job was canceled to reduce pressure, allowing all affected functionality to return to normal. Customers whose jobs failed during this window may rerun affected jobs. A small number of customers may still see workflows that appear to be running; this is actively being corrected and does not impact billing.\n\nWe thank you for your patience while our team worked on implementing a fix.\n\n## What Happened\n\n\\(all times UTC\\)\n\nAt 13:30 on July 2, 2026, as part of normal platform maintenance, our engineering team began a data deletion job, divided into several hundred individual maintenance tasks. At 14:40, unrelated to our platform maintenance, a significant number of customer project deletion requests were received and began executing.\n\nAt 15:12, the combined load from the maintenance tasks and the customer-driven projection deletions caused an internal data service to slow down, and this began cascading into our API layer. Because connections between the API layer and the workflows data service were not configured to time out slow responses under elevated load conditions, the API-handling components became overwhelmed, went unresponsive, and began failing their health checks. This caused our request routing layer to lose available upstream requests, resulting in customers seeing errors when attempting to access the UI, to log in via GitHub or Bitbucket OAuth, or to start pipelines.\n\nAt 15:16, our monitoring systems alerted and engineers were paged. The team investigated and confirmed the source of the elevated load. At 15:30 Engineers began to scale up the affected services and cancelled a long-running maintenance task to drain the load. The team noted signs of that the system was recovering around 16:12 and adjusted the status to monitoring at 16:24.\n\nAt 16:44, the team confirmed that all customer-facing services had recovered to normal operating levels. Background load from the deletion job drained by approximately 17:00. Following the incident, we discovered a small number of workflows continued to inaccurately report that they were in a running state. This should be corrected within the next few days. In all cases, customers will only be billed for actual compute time used.\n\n## Future Prevention and Process Improvement\n\nOur team is implementing the following improvements as a result of this incident:\n\n* After the incident was resolved we added timeout protections on connections between our API layer and internal data services, so that slow responses from a downstream service cannot exhaust API worker capacity.\n* We are actively investigating ways to enforce resource limits on background data tasks to prevent them from impacting customer-facing services.\n* We are improving observability and alerting on the affected data service so that resource pressure is detected earlier, before it reaches a level that impacts customers.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "critical",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-07-02T16:44:04.676Z",
"resolved_inferred": false,
"started_at": "2026-07-02T15:25:28.360Z",
"state": "postmortem",
"title": "Outage impacting web experience and pipelines starting",
"updated_at": "2026-07-10T18:57:59.887Z",
"url": "https://stspg.io/n86hwbb1whpc"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-29T16:58:09.761Z",
"resolved_inferred": false,
"started_at": "2026-06-29T10:23:34.607Z",
"state": "resolved",
"title": "Some customers are unable to access their organizations",
"updated_at": "2026-07-06T17:17:47.423Z",
"url": "https://stspg.io/ty6ll6xj9c18"
},
{
"body": "We observed a surge in demand for Xcode 26.6 and have since mitigated the incident by increasing the available capacity for that Xcode version. Customers may have seen jobs queueing for ~10-15 mins.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-26T15:00:00.000Z",
"resolved_inferred": false,
"started_at": "2026-06-26T15:00:00.000Z",
"state": "resolved",
"title": "Xcode 26.6 Demand + Queueing",
"updated_at": "2026-06-26T15:30:45.805Z",
"url": "https://stspg.io/z1sdwdk27983"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-25T23:03:58.327Z",
"resolved_inferred": false,
"started_at": "2026-06-25T22:44:15.737Z",
"state": "resolved",
"title": "Investigating \u2014 Usage API Not Available",
"updated_at": "2026-06-25T23:03:58.346Z",
"url": "https://stspg.io/0hqc49q254gd"
},
{
"body": "We are concluding this incident based on a confirmed fix with our infrastructure provider and successful tests.  For any customers seeing failed runs, please run again for a successful connection.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-25T03:06:41.792Z",
"resolved_inferred": false,
"started_at": "2026-06-24T23:13:45.020Z",
"state": "resolved",
"title": "Issues with network routing for Mac jobs",
"updated_at": "2026-07-10T14:24:46.599Z",
"url": "https://stspg.io/mrq29l3fw2cm"
},
{
"body": "## Summary\n\nOn Tuesday, June 23, 2026, our engineering team identified that a single Docker job cluster was operating below expected performance levels. Our infrastructure is organized into routing groups, each responsible for handling a specific segment of traffic, and this cluster was the sole member of its group. To address the performance issue, they initiated a standard two-step procedure to safely take it out of rotation: first, update the routing configuration to stop directing traffic to the cluster, then take it offline.\n\nWhen the team moved to take the cluster offline, the first step of the procedure was skipped. The cluster was removed from the available pool, but the routing configuration was not updated, meaning traffic continued to be directed to the routing group even though it had no capacity to process work. With no other clusters in the routing group to absorb the traffic, affected jobs had nowhere to go and began queuing rather than starting. This is a routine and safe operation when the two-step procedure is followed correctly, this incident exposed a gap in our tooling that allowed the procedure to be completed out of order.\n\nThe issue began at approximately 17:25 UTC and was fully resolved by 19:46 UTC, for a total impact window of roughly 2 hours and 20 minutes. Only Docker jobs were affected; other job types, including machine and macOS jobs, continued to operate normally.\n\nBeyond the missed step, several factors extended the impact window. An automated alert fired within a minute of the change deploying, but was dismissed as expected behavior. This delayed detection by nearly an hour. When engineers attempted to accelerate recovery by restoring a prior known-good version, they encountered unexpected issues with the rollback process, adding further delay before the fix could be deployed.\n\nThe original status page for this incident can be found [here](https://status.circleci.com/incidents/rp9y8kszn25j).\n\n## How Job Routing Works\n\nCircleCI uses Nomad as its job scheduling system for Docker-based jobs. When a job is triggered, it is routed to one of several Nomad clusters, organized into routing groups each responsible for handling a specific segment of traffic. This multi-cluster architecture is intentional: clusters are isolated from one another so that an issue in one does not impact the others, limiting blast radius and improving overall resiliency. Under normal circumstances, if a cluster becomes unavailable, traffic is automatically rerouted to the remaining operational clusters in its routing group, allowing jobs to continue without interruption.\n\n## What Happened\n\n\\(All times UTC\\)\n\nPrior to the incident, our engineering team identified that a single Docker job cluster was operating below expected performance levels. Our infrastructure is organized into routing groups, each responsible for handling a specific segment of traffic, and this cluster was the sole member of its group. To address the performance issue, they initiated a standard two-step procedure to safely take it out of rotation.\n\nAt 16:58, our engineering team applied a configuration change to remove the affected cluster from service. The first step of the two-step procedure was missed. The cluster was removed from the available pool, but the routing configuration was not updated, leaving traffic still being directed to a routing group with no capacity to process it.\n\nWhen the configuration change was fully deployed at 17:25, the affected jobs began to queue.\n\nAt 17:26, our monitoring alerted us to reduced job throughput on the affected cluster. Because throughput on that cluster was expected to drop as part of the cluster rotation procedure, the alert was interpreted as expected behavior and no action was taken.\n\nBy 18:19, a series of customer support tickets made clear that something beyond expected behavior was occurring. A formal incident was declared at 18:28 and engineers began investigating.\n\nBy 18:33, the team confirmed the impact was isolated to a subset of Docker jobs and identified a large backlog of queued jobs. At 18:41, a fix was merged to restore the affected cluster back into the available pool. Deployment of that fix was delayed because the CI smoke tests assume functioning clusters, leading to some rework of the pipeline itself. In parallel, the team attempted to accelerate recovery at 18:54 by restoring a prior known-good configuration directly, but encountered a conflict that left multiple active versions running simultaneously, preventing a clean rollback. The fix was fully deployed at 19:20 and Docker jobs began processing again.\n\nBy 19:28, the team confirmed the backlog was actively clearing and newly submitted jobs were starting at normal times.\n\nCustomers who had accumulated queued jobs during the incident period may have experienced a surge in concurrent job starts upon recovery, potentially hitting plan concurrency limits, which would have resulted in jobs still queueing while we processed the backlog.\n\nIn total, the period during which Docker jobs were unable to start lasted from approximately 17:25 to 19:20, roughly 1 hour and 55 minutes. The additional time from 19:20 to 19:46 was spent processing the accumulated backlog, during which some previously queued jobs completed with elevated wait times.\n\n## Future Prevention and Process Improvement\n\nWe are taking the following steps to prevent a recurrence and improve our response time:\n\n**We are strengthening our failover architecture.** Our multi-cluster design is built for resiliency, and while automatic failover mechanisms are already in place, this incident exposed a gap in our safeguards. We are closing that gap to ensure traffic is rerouted to healthy clusters across routing groups regardless of whether a cluster goes down unexpectedly or is intentionally taken offline.\n\n**We are enhancing our cluster rotation procedure.** This incident was possible because our tooling allowed an invalid state to exist, a cluster removed from the available pool while the routing configuration still directed traffic to it. We are updating this tooling to enforce that routing configuration and cluster availability are always updated together, and that the procedure cannot be completed out of order.\n\n**We are improving our monitoring and alerting.** The automated alert that fired during this incident was dismissed because it appeared consistent with the operation being performed. We are improving our monitoring so that unexpected side effects are surfaced clearly and are not confused with expected behavior.\n\n**We are improving our rollback capabilities.** During the incident, our team encountered delays when attempting to deploy the fix. We are auditing our deployment configurations to ensure that rollbacks can be executed more quickly and reliably during incidents.\n\nReliability is our top priority, and we are committed to ensuring that infrastructure upgrades like this one are executed with the appropriate safeguards in place.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "major",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-23T19:58:24.368Z",
"resolved_inferred": false,
"started_at": "2026-06-23T18:34:35.096Z",
"state": "postmortem",
"title": "Delays starting Docker jobs",
"updated_at": "2026-06-27T00:16:07.950Z",
"url": "https://stspg.io/dw1r4sr0mrtl"
},
{
"body": "Customers using Bitbucket as their VCS provider or as their login provider experienced errors when interacting with Bitbucket repositories and authenticating to CircleCI. This was caused by an incident with Atlassian Bitbucket's OAuth service, which has now been resolved and all affected functionality has returned to normal.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-12T00:18:31.301Z",
"resolved_inferred": false,
"started_at": "2026-06-11T21:31:49.306Z",
"state": "resolved",
"title": "Issue authenticating only Bitbucket users",
"updated_at": "2026-06-12T00:18:31.319Z",
"url": "https://stspg.io/qjt275qv8plh"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-10T23:07:35.612Z",
"resolved_inferred": false,
"started_at": "2026-06-10T22:39:15.430Z",
"state": "resolved",
"title": "Issues publishing orbs",
"updated_at": "2026-06-10T23:07:35.626Z",
"url": "https://stspg.io/zwjbxfw5b7yy"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-10T16:41:14.512Z",
"resolved_inferred": false,
"started_at": "2026-06-10T16:30:49.849Z",
"state": "resolved",
"title": "Incident with GitHub APIs",
"updated_at": "2026-06-10T16:41:14.526Z",
"url": "https://stspg.io/86hq0zrpf1mp"
},
{
"body": "Between 13:29 UTC and 18:20 UTC on June 9, 2026, some customers experienced 500 errors when making requests to the pipeline values and workflow API endpoints, as well as errors in the UI. Jobs running were not impacted during this window. The issue has been resolved and all affected functionality has returned to normal. We thank you for your patience while our team worked on implementing a fix.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-09T18:43:23.042Z",
"resolved_inferred": false,
"started_at": "2026-06-09T17:45:04.136Z",
"state": "resolved",
"title": "Increased 500 Errors on Pipeline and Workflow API Endpoints and UI",
"updated_at": "2026-06-09T18:43:23.059Z",
"url": "https://stspg.io/frwqpjlwzb9r"
},
{
"body": "This issue has been resolved and macOS jobs across all resource classes are starting normally again. Customers whose jobs failed to start during this window may rerun them. We thank you for your patience while our team worked on implementing a fix.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "major",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-07T04:44:29.255Z",
"resolved_inferred": false,
"started_at": "2026-06-07T04:20:18.650Z",
"state": "resolved",
"title": "macOS jobs failing to start",
"updated_at": "2026-06-07T04:44:29.271Z",
"url": "https://stspg.io/fcqx1c8m4dqx"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-02T13:11:15.415Z",
"resolved_inferred": false,
"started_at": "2026-06-02T12:52:06.923Z",
"state": "resolved",
"title": "Errors on Cloud Docker Pulls",
"updated_at": "2026-06-02T13:11:15.436Z",
"url": "https://stspg.io/4yrvgyvkfqpy"
},
{
"body": "All Mac jobs are running successfully.\n\nOur infrastructure provider has confirmed that the DDoS attack has been mitigated and that services have been restored.\n\nThank you for your patience during this incident.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-01T03:26:10.499Z",
"resolved_inferred": false,
"started_at": "2026-06-01T01:04:39.220Z",
"state": "resolved",
"title": "Mac jobs failing to start",
"updated_at": "2026-06-01T03:26:10.524Z",
"url": "https://stspg.io/n3brb41nn16r"
},
{
"body": "Mac jobs have now fully recovered.\n\nThank you for your patience during this incident.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-06-01T00:45:07.993Z",
"resolved_inferred": false,
"started_at": "2026-05-31T18:15:41.770Z",
"state": "resolved",
"title": "Mac jobs failing to start",
"updated_at": "2026-06-01T00:45:08.010Z",
"url": "https://stspg.io/fd7x5v1g2kw9"
},
{
"body": "The Usage API incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-05-29T18:10:15.886Z",
"resolved_inferred": false,
"started_at": "2026-05-29T14:51:42.399Z",
"state": "resolved",
"title": "Usage API data delayed for 5/28",
"updated_at": "2026-05-29T18:10:15.904Z",
"url": "https://stspg.io/vc8xbt2r7rmw"
},
{
"body": "Customers running jobs that pulled dependencies from Maven Central experienced elevated 429 (rate limit) errors. We have worked with Sonatype to allow list CircleCI's dedicated IP ranges. Customers using our IP ranges feature should no longer see Maven Central rate limits affect their jobs while we continue work to fully address the issue for all customers.\n\nCustomers whose jobs failed during this window may rerun affected jobs. We thank you for your patience while our team worked on implementing a fix.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-05-14T19:38:54.325Z",
"resolved_inferred": false,
"started_at": "2026-05-14T16:49:02.701Z",
"state": "resolved",
"title": "Elevated 429 errors pulling from maven central",
"updated_at": "2026-05-14T19:38:54.342Z",
"url": "https://stspg.io/c8h5p945vnkw"
},
{
"body": "This incident is resolved. Thank you for your patience, and we apologize for any inconvenience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "major",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-05-06T20:13:46.165Z",
"resolved_inferred": false,
"started_at": "2026-05-06T19:42:48.226Z",
"state": "resolved",
"title": "Some Pipelines are being lost",
"updated_at": "2026-05-06T20:13:46.182Z",
"url": "https://stspg.io/m852c4blmtsk"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-05-06T21:26:23.566Z",
"resolved_inferred": false,
"started_at": "2026-05-06T17:54:34.393Z",
"state": "resolved",
"title": "GitHub incident with Pull Requests",
"updated_at": "2026-05-06T21:26:23.582Z",
"url": "https://stspg.io/pttscyh3qsv1"
},
{
"body": "This incident has been resolved.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-05-01T11:08:22.351Z",
"resolved_inferred": false,
"started_at": "2026-05-01T10:51:24.148Z",
"state": "resolved",
"title": "Jobs using Xcode 14.3.1 on m4pro.large are delayed.",
"updated_at": "2026-05-01T11:08:22.366Z",
"url": "https://stspg.io/f6kcq9nb85wt"
},
{
"body": "This issue has been resolved. All data is up to date through yesterday.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-05-01T14:02:02.849Z",
"resolved_inferred": false,
"started_at": "2026-05-01T10:04:37.280Z",
"state": "resolved",
"title": "Usage API data is delayed for data from yesterday, 4/30. All prior data is available. We are investigating.",
"updated_at": "2026-05-01T14:02:06.065Z",
"url": "https://stspg.io/ld03v9l2cm5c"
},
{
"body": "Customers running macOS jobs on m4pro.medium and m4pro.large with the xcode:26.4.0 image experienced job rejections with the error: \"Job was rejected because resource class <class name>, image xcode:26.4.0 is not a valid resource class\". The issue has been resolved and affected jobs should now run normally. \n\nCustomers whose jobs failed may rerun affected jobs. \n\nWe thank you for your patience while our team worked on implementing a fix.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-04-24T18:36:22.290Z",
"resolved_inferred": false,
"started_at": "2026-04-24T17:25:16.042Z",
"state": "resolved",
"title": "Build failures Xcode 26.4.0",
"updated_at": "2026-04-24T18:36:22.307Z",
"url": "https://stspg.io/c4b1vth00hcg"
},
{
"body": "The Usage API issue has been resolved. All data is now current. Appreciate your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-04-18T15:05:36.562Z",
"resolved_inferred": false,
"started_at": "2026-04-18T12:07:48.663Z",
"state": "resolved",
"title": "Usage API data delayed this morning",
"updated_at": "2026-04-18T15:05:36.580Z",
"url": "https://stspg.io/h7yfy5vyf3dw"
},
{
"body": "The Usage API incident is resolved on an ongoing basis. There was a 4-hour period yesterday 4/7 for which we did not capture RAM and CPU utilization in our logs. We will not be backfilling the data for that period. Appreciate your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-04-08T20:33:29.716Z",
"resolved_inferred": false,
"started_at": "2026-04-08T13:20:12.689Z",
"state": "resolved",
"title": "Delays in Workflow and Job data in the Usage API",
"updated_at": "2026-04-08T20:33:29.733Z",
"url": "https://stspg.io/b6482c23v6nr"
},
{
"body": "**What happened**  \nAll modals and dialogs across the CircleCI application became invisible. They were being rendered correctly in the background, but were not visible to users. This affected approval jobs, SSH key management, and other settings pages across the app.  \n  \n**Root cause**  \nThe issue was introduced by an upgrade to our internal design system library. The new version added animation support to modal components using a library called Framer Motion. These animations are designed to fade modals in from invisible \\(`opacity: 0`\\) to visible \\(`opacity: 1`\\) when opened.  \n  \nHowever, the animation logic reads open/close state from a specific React context that is only available when modals are opened via a particular component \\(`DialogTrigger`\\). Our application manages modal state differently, passing open/close state directly via props, so that context was never populated. As a result, every modal in the app was permanently stuck in the \"hidden\" animation state and never transitioned to visible.  \n  \n**Resolution**  \nWe reverted the design system upgrade to restore the previous working version, which resolved the issue immediately. We are now working on a follow-up fix that will upgrade Framer Motion properly, by ensuring all modal components receive the required context.  \n  \n**Impact**  \nThe issue affected all modal dialogs across project settings, org settings, user settings, plans & payments, and other areas of the app for the duration of the incident. No data was lost or corrupted. The underlying functionality was intact, just inaccessible via the UI.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "major",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-04-06T21:12:35.372Z",
"resolved_inferred": false,
"started_at": "2026-04-06T20:33:28.468Z",
"state": "postmortem",
"title": "Degradation in CircleCI UI elements",
"updated_at": "2026-04-07T00:22:59.313Z",
"url": "https://stspg.io/3qm63nb96zjy"
},
{
"body": "The Usage API issue affecting 4/1 data has been resolved. Updated data will be available on 4/3. Thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-04-03T02:18:05.030Z",
"resolved_inferred": false,
"started_at": "2026-04-02T19:19:48.929Z",
"state": "resolved",
"title": "Usage API data for 4/1 is incomplete with multiple columns showing null values. Data prior to 4/1 is unaffected. We are investigating.",
"updated_at": "2026-04-03T02:18:05.047Z",
"url": "https://stspg.io/78v0qz8pybdn"
},
{
"body": "It appears that GitHub's `GET /repos/{owner}/{repo}/commits` API is now stable and responsive.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "major",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-04-01T12:23:57.585Z",
"resolved_inferred": false,
"started_at": "2026-04-01T11:55:06.957Z",
"state": "resolved",
"title": "Some pipelines not triggering due to GitHub API failures",
"updated_at": "2026-04-01T12:23:57.601Z",
"url": "https://stspg.io/3q4dxfr93jhd"
},
{
"body": "On March 18, 2026, customers using workflows configured with serial groups or terminal job dependencies experienced two brief periods of disruption due to a code change.\n\nBetween approximately 12:08 and 12:35 UTC, affected pipelines encountered errors when creating workflows. For pipelines created in a second window from approximately 14:23 to 14:35 UTC, GitHub status updates were not delivered for some completed jobs. Long-running workflows triggered during this window may encounter these errors after the resolution time.  The underlying change was reverted in both cases.\n\nAll systems have been operating normally since 14:35 UTC.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "none",
"last_seen": "2026-09-16T12:28:20Z",
"resolved_at": "2026-03-18T16:00:00.000Z",
"resolved_inferred": false,
"started_at": "2026-03-18T16:00:00.000Z",
"state": "resolved",
"title": "Pipelines \u2014 Workflow Creation Errors and Missing GitHub Status Updates",
"updated_at": "2026-03-18T17:15:56.342Z",
"url": "https://stspg.io/n8qyfjwryytr"
},
{
"body": "Our engineering team has identified the root cause and has fixes prepared. Due to the nature of the data refresh process, the fix will require a deployment followed by a full refresh cycle before data is fully up to date. We expect that the data should be available within 24-36 hours. The last-24-hours reporting window continues to work normally, and all pipeline pages and job runs remain unaffected.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-15T12:28:18Z",
"resolved_at": "2026-03-16T03:52:47.915Z",
"resolved_inferred": false,
"started_at": "2026-03-16T03:02:28.023Z",
"state": "resolved",
"title": "Insights API and Dashboard displaying stale data for reporting windows greater than 24 hours",
"updated_at": "2026-03-16T03:52:47.933Z",
"url": "https://stspg.io/hk09l57sybkn"
},
{
"body": "The issue affecting the orb list on the Organization Settings page has been resolved. Our engineers have deployed a fix and users should now be able to view their orb list as expected.\u00a0 Orb usage in pipelines and builds was not impacted during this incident. If you continue to experience issues, please contact CircleCI support.\n\nThank you for your patience while we worked on resolving this issue.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-13T12:30:53Z",
"resolved_at": "2026-03-11T02:24:16.348Z",
"resolved_inferred": false,
"started_at": "2026-03-11T01:58:22.909Z",
"state": "resolved",
"title": "Unable to view orb list on Organization Settings page",
"updated_at": "2026-03-11T02:24:16.363Z",
"url": "https://stspg.io/q09f0398btms"
},
{
"body": "Looks like everything is good. Thank you for your patience.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-10T12:15:40Z",
"resolved_at": "2026-03-02T18:21:06.106Z",
"resolved_inferred": false,
"started_at": "2026-03-02T17:59:14.000Z",
"state": "resolved",
"title": "docker not working in cimg:*",
"updated_at": "2026-03-02T18:21:06.123Z",
"url": "https://stspg.io/wqmcz2chv2f6"
},
{
"body": "GitHub have acknowledged and resolved an incident affecting their platform.\nhttps://www.githubstatus.com/incidents/vd3xqfq36rgm\nUsers should no longer experience intermittent checkout failures.",
"first_seen": "2026-09-04T07:06:16Z",
"impact": "minor",
"last_seen": "2026-09-04T14:19:03Z",
"resolved_at": "2026-02-27T00:17:31.594Z",
"resolved_inferred": false,
"started_at": "2026-02-26T22:50:46.222Z",
"state": "resolved",
"title": "Degradation in some checkout steps",
"updated_at": "2026-02-27T00:17:31.613Z",
"url": "https://stspg.io/vmpflb7sk964"
}
]
}