A load balancer you can register backends with

01Build the registry

Here is the version everybody writes first:

public final class LoadBalancer {
    private final List<Backend> backends = new ArrayList<>();
    private final AtomicInteger counter = new AtomicInteger();
 
    public void register(Backend b)     { backends.add(b); }
    public void deregister(String id)   { backends.removeIf(b -> b.id().equals(id)); }
 
    public Backend next() {
        int n = backends.size();                       // (1)
        if (n == 0) throw new IllegalStateException("no backends");
        return backends.get(counter.getAndIncrement() % n);   // (2)
    }
}

type LoadBalancer struct {
	backends []*Backend
	counter  uint64
}
 
func (lb *LoadBalancer) Register(b *Backend)  { lb.backends = append(lb.backends, b) }
func (lb *LoadBalancer) Deregister(id string) {
	out := lb.backends[:0]
	for _, b := range lb.backends {
		if b.ID != id {
			out = append(out, b)
		}
	}
	lb.backends = out
}
 
func (lb *LoadBalancer) Next() (*Backend, error) {
	n := len(lb.backends)                               // (1)
	if n == 0 {
		return nil, errors.New("no backends")
	}
	i := atomic.AddUint64(&lb.counter, 1)
	return lb.backends[int(i)%n], nil                   // (2)
}

class LoadBalancer:
    def __init__(self) -> None:
        self._backends: list[Backend] = []
        self._counter = 0
 
    def register(self, b: Backend) -> None:
        self._backends.append(b)
 
    def deregister(self, id: str) -> None:
        self._backends = [b for b in self._backends if b.id != id]
 
    def next(self) -> Backend:
        n = len(self._backends)                          # (1)
        if n == 0:
            raise RuntimeError("no backends")
        self._counter += 1
        return self._backends[self._counter % n]         # (2)

02Break it

sequenceDiagram
  participant A as next() thread
  participant L as backends
  participant R as registrar thread
  Note over L: [b0, b1, b2]
  A->>L: size()
  L-->>A: 3
  R->>L: deregister(b2)
  Note over L: [b0, b1]
  A->>A: counter=5, 5 % 3 = 2
  A->>L: get(2)
  L-->>A: IndexOutOfBounds

next() reads the list twice. It reads the size, then it reads an element. Those are two separate operations against a structure somebody else is allowed to mutate in between.

sequenceDiagram
  participant A as next() thread
  participant L as backends
  participant R as registrar thread
  Note over L: [b0, b1, b2]
  A->>L: size()
  L-->>A: 3
  R->>L: deregister(b2)
  Note over L: [b0, b1]
  A->>A: counter=5, 5 % 3 = 2
  A->>L: get(2)
  L-->>A: IndexOutOfBounds

This is the same shape as the bank's check-then-act, wearing different clothes. There the check was balance >= amount; here it is size(). In both cases the decision was made against a world that had moved on by the time you acted on it.

And notice the second, quieter failure: even when the index happens to be in range, % n with a changing n means the rotation is not round-robin at all. Remove one backend from three and every request lands somewhere different from where it would have — the mapping reshuffles rather than closing the gap.

03Fix it from the inside

public final class LoadBalancer {
    // The reference is volatile; the list it points at is never mutated.
    private volatile List<Backend> backends = List.of();
    private final AtomicInteger counter = new AtomicInteger();
    private final Object writeLock = new Object();
 
    public void register(Backend b) {
        synchronized (writeLock) {                 // writers serialise
            var next = new ArrayList<>(backends);
            next.add(b);
            backends = List.copyOf(next);          // one atomic publish
        }
    }
 
    public void deregister(String id) {
        synchronized (writeLock) {
            backends = backends.stream()
                    .filter(b -> !b.id().equals(id))
                    .toList();
        }
    }
 
    public Backend next() {
        List<Backend> snapshot = backends;         // ONE read of the reference
        int n = snapshot.size();
        if (n == 0) throw new IllegalStateException("no backends");
        return snapshot.get(Math.floorMod(counter.getAndIncrement(), n));
    }
}

type LoadBalancer struct {
	backends atomic.Pointer[[]*Backend] // never mutated in place
	counter  atomic.Uint64
	mu       sync.Mutex // writers only
}
 
func (lb *LoadBalancer) Register(b *Backend) {
	lb.mu.Lock()
	defer lb.mu.Unlock()
	cur := lb.load()
	next := make([]*Backend, len(cur), len(cur)+1)
	copy(next, cur)
	next = append(next, b)
	lb.backends.Store(&next)
}
 
func (lb *LoadBalancer) load() []*Backend {
	if p := lb.backends.Load(); p != nil {
		return *p
	}
	return nil
}
 
func (lb *LoadBalancer) Next() (*Backend, error) {
	snapshot := lb.load() // ONE load; the slice header cannot change under us
	n := len(snapshot)
	if n == 0 {
		return nil, errors.New("no backends")
	}
	i := lb.counter.Add(1)
	return snapshot[i%uint64(n)], nil
}

import itertools, threading
 
class LoadBalancer:
    def __init__(self) -> None:
        self._backends: tuple[Backend, ...] = ()   # immutable
        self._counter = itertools.count()          # thread-safe in CPython
        self._write_lock = threading.Lock()
 
    def register(self, b: Backend) -> None:
        with self._write_lock:
            self._backends = self._backends + (b,)     # rebind, never mutate
 
    def deregister(self, id: str) -> None:
        with self._write_lock:
            self._backends = tuple(b for b in self._backends if b.id != id)
 
    def next(self) -> Backend:
        snapshot = self._backends        # ONE read of the attribute
        n = len(snapshot)
        if n == 0:
            raise RuntimeError("no backends")
        return snapshot[next(self._counter) % n]

04Fix it at the boundary instead

Everything above fixes next() by being careful inside it. There is a stronger move: change the signature so carelessness is not available.

Split the two responsibilities. The registry owns the set of backends. The strategy picks one from a list it is handed:

public interface RoutingStrategy {
    Backend route(List<Backend> nodes);
}
 
public final class RoundRobinStrategy implements RoutingStrategy {
    private final AtomicInteger counter = new AtomicInteger();
 
    @Override
    public Backend route(List<Backend> nodes) {
        if (nodes.isEmpty()) throw new IllegalStateException("no backends");
        return nodes.get(Math.floorMod(counter.getAndIncrement(), nodes.size()));
    }
}

The strategy has no field holding the backends, so there is nothing it can read twice. It receives one list and both reads go to that list. The race is not fixed, it is unconstructible — and it stays unconstructible for every strategy anyone writes later, including ones that iterate the list or read the size three times.

The registry's job is then to pass a genuine snapshot:

public Backend route() {
    return strategy.route(snapshot());   // one immutable list, per call
}

The general move is worth internalising:

• A lock makes the unsafe thing safe, as long as everyone remembers to take it.
• A boundary that only ever hands out immutable values removes the need to remember.

The second one keeps working when someone who has never read this page writes the twenty-first strategy.

05Find the two bugs the boundary does not fix

Both of these survive the redesign, and both are in the code above for a reason.

counter % size breaks at overflow. AtomicInteger.getAndIncrement wraps to Integer.MIN_VALUE, and Java's % keeps the sign of the dividend:

count=2147483647   idx= 1   -> n1
count=-2147483648  idx=-2   -> ArrayIndexOutOfBoundsException
Math.floorMod(-2147483648, 3) = 1

At a few thousand requests a second that arrives in under a fortnight of uptime, as an index-out-of-bounds with no obvious cause. Math.floorMod is the fix; counter.getAndIncrement() & Integer.MAX_VALUE also works.

An empty list is a division, not an index. count % 0 throws ArithmeticException: / by zero before get is ever reached, so the error you get names arithmetic rather than backends. Check isEmpty() before the modulo, not after.

06Find the one that only shows up in production

If the registry stores backends in a hash set — ConcurrentHashMap.newKeySet() is the natural choice, since registration wants deduplication — then snapshot() has no defined order:

insertion order : [10.0.0.1, .2, .3, .4, .5]
iteration order : [10.0.0.4, .3, .5, .2, .1]

after registering .6:
before: [.4, .3, .5, .2, .1]
after : [.4, .3, .6, .5, .2, .1]      <- .6 lands in the middle

counter=2   before -> .5   after -> .6   CHANGED
counter=3   before -> .2   after -> .5   CHANGED

A new node is inserted wherever it hashes, not at the end. So registering one backend reshuffles the rotation for every position after it.

This is a nasty one to catch, because nothing looks wrong:

• Your tests pass. The distribution across backends is still even.
• No error is raised, and no metric moves.
• In production, a single deploy silently re-points a large share of traffic.

If you need stable rotation, the snapshot has to be ordered — sort by a stable id, or keep the backends in a CopyOnWriteArrayList and use the set only for the duplicate check.