job d79319d4-ffc8-495a-822c-757cf983d0af

State	FAILED
Quota	`flowmetal-demo`
Manifest	`fbac6bee1d36b88c292d2bc779710553a876a3efb0f3a7998094c506300a3d77`
Entry override	`tirefireind.us/flowmetal-demo/dispatch-fire`/`dispatch`
Worker	`worker-31d77855-2d12-42ae-b31b-df8b50ed4289`
Inbox HWM	0
Error	Traceback (most recent call last): File <builtin>, in <module> * main.star:88, in _dispatch_impl fail("dispatch exhausted %d attempts for lot %s" % (MAX_ATTEMPTS, lot["lot_id... error: dispatch exhausted 4 attempts for lot lot_91c50290c5 --> main.star:88:5 \| 88 \| fail("dispatch exhausted %d attempts for lot %s" % (MAX_ATTEMPTS, lot["lot_id"])) \| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ \|

#0 req 0 connector:http

{
  "method": "GET",
  "url": "http://tire-registry.flowmetal.svc.cluster.local:8080/lots/next"
}

#1 req 0 ok

{
  "body": {
    "condition": "flaming",
    "discovered_at": 1776755520.0,
    "lot_id": "lot_91c50290c5",
    "weight_kg": 2273.0
  },
  "headers": {
    "content-length": "93",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:12:00 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#2 req 1 connector:http

{
  "body_json": {
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#3 req 1 ok

{
  "body": {
    "incident_id": "inc_84d9103e877b",
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:12:10 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#4 req 2 sleep deadline=1776755543.114908455 duration=2s

#5 req 2 ok

null

#6 req 3 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_84d9103e877b"
}

#7 req 3 ok

{
  "body": {
    "error_kind": "dispatch_service_down",
    "incident_id": "inc_84d9103e877b",
    "message": "dispatch failed: dispatch_service_down",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "157",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:12:22 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#8 req 4 connector:http

{
  "body_json": {
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#9 req 4 ok

{
  "body": {
    "incident_id": "inc_11ca87cbcc8f",
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:12:32 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#10 req 5 sleep deadline=1776755565.533223595 duration=2s

#11 req 5 ok

null

#12 req 6 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_11ca87cbcc8f"
}

#13 req 6 ok

{
  "body": {
    "error_kind": "dispatch_service_down",
    "incident_id": "inc_11ca87cbcc8f",
    "message": "dispatch failed: dispatch_service_down",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "157",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:12:45 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#14 req 7 connector:http

{
  "body_json": {
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#15 req 7 ok

{
  "body": {
    "incident_id": "inc_5f5daade1779",
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:12:55 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#16 req 8 sleep deadline=1776755587.977069298 duration=2s

#17 req 8 ok

null

#18 req 9 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_5f5daade1779"
}

#19 req 9 ok

{
  "body": {
    "error_kind": "dispatch_service_down",
    "incident_id": "inc_5f5daade1779",
    "message": "dispatch failed: dispatch_service_down",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "157",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:13:07 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#20 req 10 connector:http

{
  "body_json": {
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "method": "POST",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents"
}

#21 req 10 ok

{
  "body": {
    "incident_id": "inc_5345469bb334",
    "lot_id": "lot_91c50290c5",
    "priority": "urgent"
  },
  "headers": {
    "content-length": "80",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:13:12 GMT",
    "server": "uvicorn"
  },
  "status": 202.0
}

#22 req 11 sleep deadline=1776755603.330736601 duration=2s

#23 req 11 ok

null

#24 req 12 connector:http

{
  "method": "GET",
  "url": "http://fire-dispatch.flowmetal.svc.cluster.local:8080/incidents/inc_5345469bb334"
}

#25 req 12 ok

{
  "body": {
    "error_kind": "crews_unavailable",
    "incident_id": "inc_5345469bb334",
    "message": "dispatch failed: crews_unavailable",
    "retryable": true,
    "status": "failed"
  },
  "headers": {
    "content-length": "149",
    "content-type": "application/json",
    "date": "Tue, 21 Apr 2026 07:13:23 GMT",
    "server": "uvicorn"
  },
  "status": 200.0
}

#26 FAILED worker=worker-31d77855-2d12-42ae-b31b-df8b50ed4289

job d79319d4-ffc8-495a-822c-757cf983d0af

Log