#0
print
>>> [noc] pulling next host record off laskey…
#2
req 1
connector:http
{
"method": "GET",
"url": "http://laskey.flowmetal.svc.cluster.local:8080/hosts/next"
}
#3
req 1
ok
{
"body": {
"config": {
"disk_layout": "zfs-mirror",
"os": "debian-12",
"profile": "api-worker",
"rack": "rack-c1"
},
"hostname": "host-10d30fe3"
},
"headers": {
"content-length": "123",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:00 GMT",
"server": "uvicorn"
},
"status": 200.0
}
#4
print
>>> [noc] target host host-10d30fe3 (debian-12 / api-worker / zfs-mirror) — handing off to wilson.
#6
print
>>> [noc] attempt 1/4: kicking off reinstall of host-10d30fe3.
#8
req 4
connector:http
{
"body_json": {
"config": {
"disk_layout": "zfs-mirror",
"os": "debian-12",
"profile": "api-worker",
"rack": "rack-c1"
},
"hostname": "host-10d30fe3"
},
"method": "POST",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall"
}
#9
req 4
ok
{
"body": {
"hostname": "host-10d30fe3",
"reinstall_id": "ri_5745826730a8"
},
"headers": {
"content-length": "61",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:00 GMT",
"server": "uvicorn"
},
"status": 202.0
}
#10
req 5
sleep
deadline=1776766448.075849982
duration=2s
#12
req 6
connector:http
{
"method": "GET",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall/ri_5745826730a8"
}
#13
req 6
ok
{
"body": {
"eta_seconds": 4.22,
"hostname": "host-10d30fe3",
"reinstall_id": "ri_5745826730a8",
"status": "pending"
},
"headers": {
"content-length": "99",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:08 GMT",
"server": "uvicorn"
},
"status": 200.0
}
#14
req 7
sleep
deadline=1776766459.043895960
duration=2s
#16
req 8
connector:http
{
"method": "GET",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall/ri_5745826730a8"
}
#17
req 8
ok
{
"body": {
"error_kind": "wilson_forgot",
"hostname": "host-10d30fe3",
"message": "wilson forgot this reinstall ever happened",
"reinstall_id": "ri_5745826730a8",
"retryable": true,
"status": "forgotten"
},
"headers": {
"content-length": "183",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:19 GMT",
"server": "uvicorn"
},
"status": 200.0
}
#18
print
>>> [noc] attempt 1: wilson claims it never heard of ri_5745826730a8. classic.
#20
print
>>> [noc] attempt 2/4: kicking off reinstall of host-10d30fe3.
#22
req 11
connector:http
{
"body_json": {
"config": {
"disk_layout": "zfs-mirror",
"os": "debian-12",
"profile": "api-worker",
"rack": "rack-c1"
},
"hostname": "host-10d30fe3"
},
"method": "POST",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall"
}
#23
req 11
ok
{
"body": {
"hostname": "host-10d30fe3",
"reinstall_id": "ri_5df3556d4d81"
},
"headers": {
"content-length": "61",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:21 GMT",
"server": "uvicorn"
},
"status": 202.0
}
#24
req 12
sleep
deadline=1776766471.958679182
duration=2s
#26
req 13
connector:http
{
"method": "GET",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall/ri_5df3556d4d81"
}
#27
req 13
ok
{
"body": {
"error_kind": "reinstall_broken",
"hostname": "host-10d30fe3",
"message": "reinstall broke partway through",
"reinstall_id": "ri_5df3556d4d81",
"retryable": true,
"status": "failed"
},
"headers": {
"content-length": "172",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:32 GMT",
"server": "uvicorn"
},
"status": 200.0
}
#28
print
>>> [noc] attempt 2 failed (reinstall_broken); looping back to wilson.
#30
print
>>> [noc] attempt 3/4: kicking off reinstall of host-10d30fe3.
#32
req 16
connector:http
{
"body_json": {
"config": {
"disk_layout": "zfs-mirror",
"os": "debian-12",
"profile": "api-worker",
"rack": "rack-c1"
},
"hostname": "host-10d30fe3"
},
"method": "POST",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall"
}
#33
req 16
ok
{
"body": {
"hostname": "host-10d30fe3",
"reinstall_id": "ri_e41dc598ec28"
},
"headers": {
"content-length": "61",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:39 GMT",
"server": "uvicorn"
},
"status": 202.0
}
#34
req 17
sleep
deadline=1776766489.904621230
duration=2s
#36
req 18
connector:http
{
"method": "GET",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall/ri_e41dc598ec28"
}
#37
req 18
ok
{
"body": {
"error_kind": "reinstall_broken",
"hostname": "host-10d30fe3",
"message": "reinstall broke partway through",
"reinstall_id": "ri_e41dc598ec28",
"retryable": true,
"status": "failed"
},
"headers": {
"content-length": "172",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:50 GMT",
"server": "uvicorn"
},
"status": 200.0
}
#38
print
>>> [noc] attempt 3 failed (reinstall_broken); looping back to wilson.
#40
print
>>> [noc] attempt 4/4: kicking off reinstall of host-10d30fe3.
#42
req 21
connector:http
{
"body_json": {
"config": {
"disk_layout": "zfs-mirror",
"os": "debian-12",
"profile": "api-worker",
"rack": "rack-c1"
},
"hostname": "host-10d30fe3"
},
"method": "POST",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall"
}
#43
req 21
ok
{
"body": {
"hostname": "host-10d30fe3",
"reinstall_id": "ri_8544fca85640"
},
"headers": {
"content-length": "61",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:14:58 GMT",
"server": "uvicorn"
},
"status": 202.0
}
#44
req 22
sleep
deadline=1776766503.024905228
duration=2s
#46
req 23
connector:http
{
"method": "GET",
"url": "http://wilson.flowmetal.svc.cluster.local:8080/reinstall/ri_8544fca85640"
}
#47
req 23
ok
{
"body": {
"hostname": "host-10d30fe3",
"reinstall_id": "ri_8544fca85640",
"status": "succeeded",
"took_seconds": 3.12
},
"headers": {
"content-length": "102",
"content-type": "application/json",
"date": "Tue, 21 Apr 2026 10:15:03 GMT",
"server": "uvicorn"
},
"status": 200.0
}
#48
print
>>> [noc] host-10d30fe3 reinstalled clean on attempt 4. standing down.
#50
DONE
worker=worker-cb363943-e8d5-4544-959e-0acbf94251c1