Skip to content
This repository has been archived by the owner on Aug 26, 2024. It is now read-only.

make TX and RX could be in seperate go routines #12

Merged
merged 1 commit into from
Jul 12, 2021
Merged

make TX and RX could be in seperate go routines #12

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 55 additions & 30 deletions xdp.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,22 +135,22 @@ type rxTxRing struct {

// A Socket is an implementation of the AF_XDP Linux socket type for reading packets from a device.
type Socket struct {
fd int
umem []byte
fillRing umemRing
rxRing rxTxRing
txRing rxTxRing
completionRing umemRing
qidconfMap *ebpf.Map
xsksMap *ebpf.Map
program *ebpf.Program
ifindex int
numTransmitted int
numFilled int
freeDescs []bool
options SocketOptions
rxDescs []Desc
getDescs []Desc
fd int
umem []byte
fillRing umemRing
rxRing rxTxRing
txRing rxTxRing
completionRing umemRing
qidconfMap *ebpf.Map
xsksMap *ebpf.Map
program *ebpf.Program
ifindex int
numTransmitted int
numFilled int
freeRXDescs, freeTXDescs []bool
options SocketOptions
rxDescs []Desc
getTXDescs, getRXDescs []Desc
}

// SocketOptions are configuration settings used to bind an XDP socket.
Expand Down Expand Up @@ -380,11 +380,16 @@ func NewSocket(Ifindex int, QueueID int, options *SocketOptions) (xsk *Socket, e
return nil, fmt.Errorf("syscall.Bind SockaddrXDP failed: %v", err)
}

xsk.freeDescs = make([]bool, options.NumFrames)
for i := 0; i < options.NumFrames; i++ {
xsk.freeDescs[i] = true
xsk.freeRXDescs = make([]bool, options.NumFrames)
xsk.freeTXDescs = make([]bool, options.NumFrames)
for i := range xsk.freeRXDescs {
xsk.freeRXDescs[i] = true
}
xsk.getDescs = make([]Desc, 0, options.NumFrames)
for i := range xsk.freeTXDescs {
xsk.freeTXDescs[i] = true
}
xsk.getTXDescs = make([]Desc, 0, options.CompletionRingNumDescs)
xsk.getRXDescs = make([]Desc, 0, options.FillRingNumDescs)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This change to use FillRing / CompletionRing when creating the desc slices is interesting.

Still trying to reason about what it changes -- for instance, should NumFrames == CompletionRingNumDescs + FillRingNumDescs / or are they different? Do we still need a separate NumFrames option, ... ? Need to review AF_XDP docs.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

right, this PR is based on assumption that NumFrames == CompletionRingNumDescs + FillRingNumDescs; if NumFrames < CompletionRingNumDescs + FillRingNumDescs, then I don't think it gona work since NumFrames is the max number of available decs for both TX and RX; on theory NumFrames could > CompletionRingNumDescs + FillRingNumDescs, but I don't see the use case of that


return xsk, nil
}
Expand All @@ -403,7 +408,7 @@ func (xsk *Socket) Fill(descs []Desc) int {
for _, desc := range descs {
xsk.fillRing.Descs[prod&uint32(xsk.options.FillRingNumDescs-1)] = desc.Addr
prod++
xsk.freeDescs[desc.Addr/uint64(xsk.options.FrameSize)] = false
xsk.freeRXDescs[desc.Addr/uint64(xsk.options.FrameSize)] = false
}
//fencer.SFence()
*xsk.fillRing.Producer = prod
Expand All @@ -427,7 +432,7 @@ func (xsk *Socket) Receive(num int) []Desc {
for i := 0; i < num; i++ {
descs = append(descs, xsk.rxRing.Descs[cons&uint32(xsk.options.RxRingNumDescs-1)])
cons++
xsk.freeDescs[descs[i].Addr/uint64(xsk.options.FrameSize)] = true
xsk.freeRXDescs[descs[i].Addr/uint64(xsk.options.FrameSize)] = true
}
//fencer.MFence()
*xsk.rxRing.Consumer = cons
Expand All @@ -451,7 +456,7 @@ func (xsk *Socket) Transmit(descs []Desc) (numSubmitted int) {
for _, desc := range descs {
xsk.txRing.Descs[prod&uint32(xsk.options.TxRingNumDescs-1)] = desc
prod++
xsk.freeDescs[desc.Addr/uint64(xsk.options.FrameSize)] = false
xsk.freeTXDescs[desc.Addr/uint64(xsk.options.FrameSize)] = false
}
//fencer.SFence()
*xsk.txRing.Producer = prod
Expand Down Expand Up @@ -529,14 +534,34 @@ func (xsk *Socket) Poll(timeout int) (numReceived int, numCompleted int, err err
}

// GetDescs returns up to n descriptors which are not currently in use.
func (xsk *Socket) GetDescs(n int) []Desc {
if n > len(xsk.freeDescs) {
n = len(xsk.freeDescs)
// if rx is true, return desc in first half of umem, 2nd half otherwise
func (xsk *Socket) GetDescs(n int, rx bool) []Desc {
if n > cap(xsk.getRXDescs) {
n = cap(xsk.getRXDescs)
Copy link
Contributor

@crandles crandles Jun 21, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i notice you're still accessing the RXDesc fields if rx == false (here and in the following code) before referencing the TX vars.

I think it might read better to only reference the appropriate fields (depending on rx):

  • if rx { ... } else { .... }

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume you mean line 558

start = cap(xsk.getRXDescs)

this is to determine where is start index of TX, the idea is both freeTXDescs and freeRXDescs has length of options.NumFrames, but freeRXDescs only uses index 0 to options.FillRingNumDescs and freeTXDescs only use options.FillRingNumDesc onward;
the upside of this approach is not introducing too much changes to the code, the downside is of course part of freeTXDescs and freeRXDescs are never used; but given these twojust bool slice, so memory waste here is quite small

}
if !rx {
if n > cap(xsk.getTXDescs) {
n = cap(xsk.getTXDescs)
}
}
descs := xsk.getDescs[:0]
// numOfUMEMChunks := len(xsk.freeRXDescs) / 2
// if n > numOfUMEMChunks {
// n = numOfUMEMChunks
// }

descs := xsk.getRXDescs[:0]
j := 0
for i := 0; i < len(xsk.freeDescs) && j < n; i++ {
if xsk.freeDescs[i] == true {
start := 0
end := cap(xsk.getRXDescs)
freeList := xsk.freeRXDescs
if !rx {
start = cap(xsk.getRXDescs)
end = len(xsk.freeTXDescs)
freeList = xsk.freeTXDescs
descs = xsk.getTXDescs[:0]
}
for i := start; i < end && j < n; i++ {
if freeList[i] == true {
descs = append(descs, Desc{
Addr: uint64(i) * uint64(xsk.options.FrameSize),
Len: uint32(xsk.options.FrameSize),
Expand Down Expand Up @@ -613,7 +638,7 @@ func (xsk *Socket) Complete(n int) {
for i := 0; i < n; i++ {
addr := xsk.completionRing.Descs[cons&uint32(xsk.options.CompletionRingNumDescs-1)]
cons++
xsk.freeDescs[addr/uint64(xsk.options.FrameSize)] = true
xsk.freeTXDescs[addr/uint64(xsk.options.FrameSize)] = true
}
//fencer.MFence()
*xsk.completionRing.Consumer = cons
Expand Down