job 5cf31a89-acb4-44b3-ba0b-f1f28bfd090a

State	FAILED
Quota	`flowmetal-demo`
Manifest	`7da111f84daf1e10fcc12a35e8cf5c5debb80884c697bd0a1d517c10b3af1b28`
Entry override	`tirefireind.us/flowmetal-demo/dispatch-fire`/`dispatch`
Worker	`worker-5879336e-a44f-45a2-ad04-4943fa33f428`
Inbox HWM	0
Error	Traceback (most recent call last): File <builtin>, in <module> * main.star:84, in _dispatch_impl body = _open_and_poll(fctx, lot, priority, attempt) * main.star:61, in _open_and_poll body = fctx.actions.http( error: action failed: HTTP 503 (source connector) --> main.star:61:16 \| 61 \| body = fctx.actions.http( \| ________________^ 62 \| \| "GET", FIRE_DISPATCH + "/incidents/" + incident_id, 63 \| \| )["body"] \| \|_________^ \|

#0 print >>> [dispatch] phoning tire-registry for the next incoming report…

#2 req 1 connector:http

{
  "method": "GET",
  "url": "http://tire-registry.flowmetal.svc.cluster.local:8080/lots/next"
}

#3 req 1 ok

{
  "body": {
    "condition": "smoldering",
    "discovered_at": 1776775680.0,
    "lot_id": "lot_ab709bedeb",
    "weight_kg": 625.0
  },
  "headers": {
    "content-length": "95",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:48:00 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#4 print >>> [dispatch] lot lot_ab709bedeb at 625 kg, condition 'smoldering' — priority normal.

#6 print >>> [attempt 1] calling fire-dispatch to roll a containment crew…

#8 req 4 connector:http

{
  "body_json": {
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#9 req 4 ok

{
  "body": {
    "incident_id": "inc_e0c09d3732ae",
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:48:07 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#10 print >>> [attempt 1] incident inc_e0c09d3732ae open, waiting on containment…

#12 req 6 sleep deadline=1776775698.000532770 duration=2s

#13 req 6 ok

null

#14 req 7 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_e0c09d3732ae"
}

#15 req 7 ok

{
  "body": {
    "crew_eta_seconds": 4.01,
    "incident_id": "inc_e0c09d3732ae",
    "priority": "normal",
    "status": "pending"
  },
  "headers": {
    "content-length": "97",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:48:18 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#16 req 8 sleep deadline=1776775709.155286983 duration=2s

#17 req 8 ok

null

#18 req 9 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_e0c09d3732ae"
}

#19 req 9 ok

{
  "body": {
    "error_kind": "dispatch_service_down",
    "incident_id": "inc_e0c09d3732ae",
    "message": "dispatch failed: dispatch_service_down",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "157",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:48:29 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#20 print >>> [dispatch] attempt 1 failed (dispatch_service_down); retrying.

#22 req 11 sleep deadline=1776775719.094982423 duration=1s

#23 req 11 ok

null

#24 print >>> [attempt 2] calling fire-dispatch to roll a containment crew…

#26 req 13 connector:http

{
  "body_json": {
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#27 req 13 ok

{
  "body": {
    "incident_id": "inc_5422f14b08ac",
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:48:39 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#28 print >>> [attempt 2] incident inc_5422f14b08ac open, waiting on containment…

#30 req 15 sleep deadline=1776775730.021330686 duration=2s

#31 req 15 ok

null

#32 req 16 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_5422f14b08ac"
}

#33 req 16 ok

{
  "body": {
    "crew_eta_seconds": 2.32,
    "incident_id": "inc_5422f14b08ac",
    "priority": "normal",
    "status": "pending"
  },
  "headers": {
    "content-length": "97",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:48:50 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#34 req 17 sleep deadline=1776775738.130739960 duration=2s

#35 req 17 ok

null

#36 req 18 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_5422f14b08ac"
}

#37 req 18 ok

{
  "body": {
    "error_kind": "crews_unavailable",
    "incident_id": "inc_5422f14b08ac",
    "message": "dispatch failed: crews_unavailable",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "149",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:48:58 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#38 print >>> [dispatch] attempt 2 failed (crews_unavailable); retrying.

#40 req 20 sleep deadline=1776775741.984124919 duration=1s

#41 req 20 ok

null

#42 print >>> [attempt 3] calling fire-dispatch to roll a containment crew…

#44 req 22 connector:http

{
  "body_json": {
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#45 req 22 ok

{
  "body": {
    "incident_id": "inc_53241b48e904",
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:49:02 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#46 print >>> [attempt 3] incident inc_53241b48e904 open, waiting on containment…

#48 req 24 sleep deadline=1776775746.034953151 duration=2s

#49 req 24 ok

null

#50 req 25 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_53241b48e904"
}

#51 req 25 ok

{
  "body": {
    "error_kind": "dispatch_service_down",
    "incident_id": "inc_53241b48e904",
    "message": "dispatch failed: dispatch_service_down",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "157",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:49:06 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#52 print >>> [dispatch] attempt 3 failed (dispatch_service_down); retrying.

#54 req 27 sleep deadline=1776775754.950061334 duration=2s

#55 req 27 ok

null

#56 print >>> [attempt 4] calling fire-dispatch to roll a containment crew…

#58 req 29 connector:http

{
  "body_json": {
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#59 req 29 ok

{
  "body": {
    "incident_id": "inc_9f8df2c2ea40",
    "lot_id": "lot_ab709bedeb",
    "priority": "normal"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:49:15 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#60 print >>> [attempt 4] incident inc_9f8df2c2ea40 open, waiting on containment…

#62 req 31 sleep deadline=1776775763.067081243 duration=2s

#63 req 31 ok

null

#64 req 32 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_9f8df2c2ea40"
}

#65 req 32 err source=connector HTTP 503

{
  "body": {
    "detail": "dispatch board flickering"
  },
  "error_kind": "http_status",
  "headers": {
    "content-length": "38",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 12:49:22 GMT",
    "server": "uvicorn"
  },
  "status": 503.0
}

#66 FAILED worker=worker-5879336e-a44f-45a2-ad04-4943fa33f428

job 5cf31a89-acb4-44b3-ba0b-f1f28bfd090a

Log