ceph osd header_read latency very high

[liuyangovo]

OSD uses NVME as the data disk，Nvme uses PCIe 4 protocol to write 4k files from the bare disk to 700k IOPS, but the Ceph cluster has a particularly slow write speed due to OSD header_read latency issues，may I ask if there are any parameter restrictions.

root@cepha:/home/ennocad# rados bench -p nvme_test 600 write -b 4K --no-cleanup
hints = 1
Maintaining 16 concurrent writes of 4096 bytes to objects of size 4096 for up to 600 seconds or 0 objects
Object prefix: benchmark_data_cepha_2949070
  sec Cur ops   started  finished  avg MB/s  cur MB/s last lat(s)  avg lat(s)
    0       0         0         0         0         0           -           0
    1      16       231       215  0.839786  0.839844    0.117378   0.0608986
    2      16       241       225   0.43941 0.0390625    0.241016   0.0707112
    3      16      8938      8922    11.616   33.9727  0.00263358  0.00537409
    4      16     17240     17224   16.8185   32.4297  0.00160052  0.00371398
    5      16     26996     26980   21.0758   38.1094  0.00175884  0.00296359
    6      16     37280     37264   24.2577   40.1719  0.00130634  0.00257449
    7      16     47436     47420   26.4591   39.6719   0.0012293  0.00236039
    8      16     57881     57865   28.2512   40.8008  0.00150044  0.00221064
    9      16     68185     68169   29.5838     40.25  0.00230282  0.00211113
   10      16     78541     78525   30.6702   40.4531  0.00163077   0.0020363
   11      16     89074     89058    31.622   41.1445  0.00113723  0.00197509
   12      16     99277     99261   32.3077   39.8555  0.00126091  0.00193308
   13      16    109377    109361    32.857   39.4531 0.000885894  0.00190081
   14      16    119804    119788   33.4191   40.7305  0.00156693  0.00186883

osd dump_historic_ops_by_duration

ceph daemon osd.5 dump_historic_ops_by_duration
"description": "osd_op(client.279306.0:7 19.2e 19:76f645a8:::benchmark_data_cepha_2948600_object6:head [set-alloc-hint object_size 4096 write_size 4096,write 0~4096] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e3749)",
            "initiated_at": "2025-02-21T09:49:33.056019+0000",
            "age": 413.68509587900002,
            "duration": 1.40297399,
            "type_data": {
                "flag_point": "commit sent; apply or cleanup",
                "client_info": {
                    "client": "client.279306",
                    "client_addr": "172.16.2.51:0/2579560761",
                    "tid": 7
                },
                "events": [
                    {
                        "event": "initiated",
                        "time": "2025-02-21T09:49:33.056019+0000",
                        "duration": 0
                    },
                    {
                        "event": "throttled",
                        "time": "2025-02-21T09:49:33.056019+0000",
                        "duration": 0
                    },

                    **{
                        "event": "header_read",
                        "time": "2025-02-21T09:49:33.056019+0000",
                        "duration": 4294967296
                    },**

                    {
                        "event": "all_read",
                        "time": "2025-02-21T09:49:33.056026+0000",
                        "duration": 6.6429999999999998e-06
                    },
                    {
                        "event": "dispatched",
                        "time": "2025-02-21T09:49:33.056026+0000",
                        "duration": 1.4000000000000001e-07
                    },
                    {
                        "event": "queued_for_pg",
                        "time": "2025-02-21T09:49:33.056027+0000",
                        "duration": 1.373e-06
                    },
                    {
                        "event": "reached_pg",
                        "time": "2025-02-21T09:49:33.056126+0000",
                        "duration": 9.8306000000000004e-05
                    },
                    {
                        "event": "started",
                        "time": "2025-02-21T09:49:33.056652+0000",
                        "duration": 0.00052663500000000002
                    },
                    {
                        "event": "waiting for subops from 6,7",
                        "time": "2025-02-21T09:49:33.057269+0000",
                        "duration": 0.000616254
                    },
                    {
                        "event": "sub_op_commit_rec",
                        "time": "2025-02-21T09:49:34.173429+0000",
                        "duration": 1.1161609100000001
                    },
                    {
                        "event": "op_commit",
                        "time": "2025-02-21T09:49:34.173536+0000",
                        "duration": 0.000106111
                    },
                    {
                        "event": "sub_op_commit_rec",
                        "time": "2025-02-21T09:49:34.458951+0000",
                        "duration": 0.28541513800000001
                    },
                    {
                        "event": "commit_sent",
                        "time": "2025-02-21T09:49:34.458970+0000",
                        "duration": 1.9207000000000001e-05
                    },
                    {
                        "event": "done",
                        "time": "2025-02-21T09:49:34.458993+0000",
                        "duration": 2.3464000000000001e-05
                    }
                ]
            }
        },

ceph daemon /var/run/ceph/7f0ede26-e5e1-11ef-bccf-192f15c9e419/ceph-osd.5.asok dump_ops_in_flight
{
            "description": "osd_op(client.279342.0:13428 19.3a 19:5e2301ca:::benchmark_data_cepha_2958544_object13427:head [set-alloc-hint object_size 4096 write_size 4096,write 0~4096 in=4096b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e3749)",
            "initiated_at": "2025-02-21T10:16:32.539827+0000",
            "age": 0.018901068,
            "duration": 0.019000535999999998,
            "type_data": {
                "flag_point": "waiting for sub ops",
                "client_info": {
                    "client": "client.279342",
                    "client_addr": "172.16.2.51:0/428589280",
                    "tid": 13428
                },
                "events": [
                    {
                        "event": "initiated",
                        "time": "2025-02-21T10:16:32.539827+0000",
                        "duration": 0
                    },
                    {
                        "event": "throttled",
                        "time": "2025-02-21T10:16:32.539827+0000",
                        "duration": 0
                    },
                    {
                        "event": "header_read",
                        "time": "2025-02-21T10:16:32.539826+0000",
                        "duration": 4294967295.999999
                    },
                    {
                        "event": "all_read",
                        "time": "2025-02-21T10:16:32.539832+0000",
                        "duration": 5.8409999999999996e-06
                    },
                    {
                        "event": "dispatched",
                        "time": "2025-02-21T10:16:32.539832+0000",
                        "duration": 2.4999999999999999e-07
                    },
                    {
                        "event": "queued_for_pg",
                        "time": "2025-02-21T10:16:32.539834+0000",
                        "duration": 1.5629999999999999e-06
                    },
                    {
                        "event": "reached_pg",
                        "time": "2025-02-21T10:16:32.539880+0000",
                        "duration": 4.6108e-05
                    },
                    {
                        "event": "started",
                        "time": "2025-02-21T10:16:32.539997+0000",
                        "duration": 0.00011770200000000001
                    },
                    {
                        "event": "waiting for subops from 6,7",
                        "time": "2025-02-21T10:16:32.540032+0000",
                        "duration": 3.4365e-05
                    },
                    {
                        "event": "op_commit",
                        "time": "2025-02-21T10:16:32.540483+0000",
                        "duration": 0.000451113
                    },
                    {
                        "event": "sub_op_commit_rec",
                        "time": "2025-02-21T10:16:32.540552+0000",
                        "duration": 6.8999999999999997e-05
                    }
                ]
            }
        }