I think I solved my synchronous problem. I added 3 function to my shared models:
    def cum_grads(self):
        for name, p in self.named_parameters():
            if p.grad is not None:
                val = self.__getattr__(name+'_grad')
                val += p.grad.data
                self.__setattr__(name+'_grad', val)

    def reset_grads(self):
        self.zero_grad()
        for name, p in self.named_parameters():
            if p.grad is not None:
                val = self.__getattr__(name+'_grad')
                val = p.grad.data
                self.__setattr__(name+'_grad', val)

    def synchronize(self):
        for name, p in self.named_parameters():
            val = self.__getattr__(name+'_grad')
            p._grad = Variable(val)

When a worker have computed a new loss, I use cum_grads:
ensure_shared_grads(policy, shared_p)
shared_p.cum_grads()
counter.increment()

And the chief just loops as follow:
def chief(rank, params, traffic_light, counter, shared_p, shared_v, optimizer_p, optimizer_v):
    while True:
        time.sleep(1)
        # workers will wait after last loss computation
        if counter.get() > params.update_treshold:
            shared_p.synchronize()
            shared_v.synchronize()
            optimizer_p.step()
            optimizer_v.step()
            counter.reset()
            shared_p.reset_grads()
            shared_v.reset_grads()
            traffic_light.switch() # workers start new loss computation