job 60229534-4d50-4d09-8d93-303f919a13cd

State	FAILED
Quota	`flowmetal-demo`
Manifest	`7da111f84daf1e10fcc12a35e8cf5c5debb80884c697bd0a1d517c10b3af1b28`
Entry override	`tirefireind.us/flowmetal-demo/dispatch-fire`/`dispatch`
Worker	`worker-5879336e-a44f-45a2-ad04-4943fa33f428`
Inbox HWM	0
Error	Traceback (most recent call last): File <builtin>, in <module> * main.star:84, in _dispatch_impl body = _open_and_poll(fctx, lot, priority, attempt) * main.star:61, in _open_and_poll body = fctx.actions.http( error: action failed: HTTP 503 (source connector) --> main.star:61:16 \| 61 \| body = fctx.actions.http( \| ________________^ 62 \| \| "GET", FIRE_DISPATCH + "/incidents/" + incident_id, 63 \| \| )["body"] \| \|_________^ \|

#0 print >>> [dispatch] phoning tire-registry for the next incoming report…

#2 req 1 connector:http

{
  "method": "GET",
  "url": "http://tire-registry.flowmetal.svc.cluster.local:8080/lots/next"
}

#3 req 1 ok

{
  "body": {
    "condition": "smoldering",
    "discovered_at": 1776765660.0,
    "lot_id": "lot_6695358703",
    "weight_kg": 317.0
  },
  "headers": {
    "content-length": "95",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:01:00 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#4 print >>> [dispatch] lot lot_6695358703 at 317 kg, condition 'smoldering' — priority normal.

#6 print >>> [attempt 1] calling fire-dispatch to roll a containment crew…

#8 req 4 connector:http

{
  "body_json": {
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#9 req 4 ok

{
  "body": {
    "incident_id": "inc_7c2fef12b427",
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:01:03 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#10 print >>> [attempt 1] incident inc_7c2fef12b427 open, waiting on containment…

#12 req 6 sleep deadline=1776765671.083031280 duration=2s

#13 req 6 ok

null

#14 req 7 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_7c2fef12b427"
}

#15 req 7 ok

{
  "body": {
    "crew_eta_seconds": 0.76,
    "incident_id": "inc_7c2fef12b427",
    "priority": "normal",
    "status": "pending"
  },
  "headers": {
    "content-length": "97",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:01:11 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#16 req 8 sleep deadline=1776765682.084783757 duration=2s

#17 req 8 ok

null

#18 req 9 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_7c2fef12b427"
}

#19 req 9 ok

{
  "body": {
    "error_kind": "dispatch_service_down",
    "incident_id": "inc_7c2fef12b427",
    "message": "dispatch failed: dispatch_service_down",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "157",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:01:22 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#20 print >>> [dispatch] attempt 1 failed (dispatch_service_down); retrying.

#22 req 11 sleep deadline=1776765694.006421797 duration=1s

#23 req 11 ok

null

#24 print >>> [attempt 2] calling fire-dispatch to roll a containment crew…

#26 req 13 connector:http

{
  "body_json": {
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#27 req 13 ok

{
  "body": {
    "incident_id": "inc_e545b73d56f7",
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:01:34 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#28 print >>> [attempt 2] incident inc_e545b73d56f7 open, waiting on containment…

#30 req 15 sleep deadline=1776765707.276674886 duration=2s

#31 req 15 ok

null

#32 req 16 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_e545b73d56f7"
}

#33 req 16 ok

{
  "body": {
    "error_kind": "dispatch_service_down",
    "incident_id": "inc_e545b73d56f7",
    "message": "dispatch failed: dispatch_service_down",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "157",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:01:47 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#34 print >>> [dispatch] attempt 2 failed (dispatch_service_down); retrying.

#36 req 18 sleep deadline=1776765719.057229837 duration=1s

#37 req 18 ok

null

#38 print >>> [attempt 3] calling fire-dispatch to roll a containment crew…

#40 req 20 connector:http

{
  "body_json": {
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#41 req 20 ok

{
  "body": {
    "incident_id": "inc_524972137181",
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:01:59 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#42 print >>> [attempt 3] incident inc_524972137181 open, waiting on containment…

#44 req 22 sleep deadline=1776765722.916713621 duration=2s

#45 req 22 ok

null

#46 req 23 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_524972137181"
}

#47 req 23 ok

{
  "body": {
    "crew_eta_seconds": 9.59,
    "incident_id": "inc_524972137181",
    "priority": "normal",
    "status": "pending"
  },
  "headers": {
    "content-length": "97",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:02:03 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#48 req 24 sleep deadline=1776765731.171239835 duration=2s

#49 req 24 ok

null

#50 req 25 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_524972137181"
}

#51 req 25 ok

{
  "body": {
    "crew_eta_seconds": 1.56,
    "incident_id": "inc_524972137181",
    "priority": "normal",
    "status": "pending"
  },
  "headers": {
    "content-length": "97",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:02:11 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#52 req 26 sleep deadline=1776765740.144933138 duration=2s

#53 req 26 ok

null

#54 req 27 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_524972137181"
}

#55 req 27 ok

{
  "body": {
    "error_kind": "crews_unavailable",
    "incident_id": "inc_524972137181",
    "message": "dispatch failed: crews_unavailable",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "149",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:02:20 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#56 print >>> [dispatch] attempt 3 failed (crews_unavailable); retrying.

#58 req 29 sleep deadline=1776765750.912012403 duration=2s

#59 req 29 ok

null

#60 print >>> [attempt 4] calling fire-dispatch to roll a containment crew…

#62 req 31 connector:http

{
  "body_json": {
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#63 req 31 ok

{
  "body": {
    "incident_id": "inc_80da99950e4e",
    "lot_id": "lot_6695358703",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:02:31 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#64 print >>> [attempt 4] incident inc_80da99950e4e open, waiting on containment…

#66 req 33 sleep deadline=1776765759.027128726 duration=2s

#67 req 33 ok

null

#68 req 34 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_80da99950e4e"
}

#69 req 34 ok

{
  "body": {
    "crew_eta_seconds": 3.53,
    "incident_id": "inc_80da99950e4e",
    "priority": "normal",
    "status": "pending"
  },
  "headers": {
    "content-length": "97",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:02:39 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#70 req 35 sleep deadline=1776765769.979886604 duration=2s

#71 req 35 ok

null

#72 req 36 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_80da99950e4e"
}

#73 req 36 err source=connector HTTP 503

{
  "body": {
    "detail": "dispatch board flickering"
  },
  "error_kind": "http_status",
  "headers": {
    "content-length": "38",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 10:02:50 GMT",
    "server": "uvicorn"
  },
  "status": 503.0
}

#74 FAILED worker=worker-5879336e-a44f-45a2-ad04-4943fa33f428

job 60229534-4d50-4d09-8d93-303f919a13cd

Log