[Python] Using ray to parallelize simulator python

I'm new to ray and I'm trying to parallelize the simulator that I have developed. Here an example of my simulator obviously its more complex.

import some_library
import sim_library_with_global_object

class Model(object):
def __init__(self,init_vals):
#initialize object using some of the global_object from sim_library.
#the Model object have it's own variables not global

def do_step(self,time):
#calculate Model step using the global_object from sim_library
#edit the Model variables with respect to the step


class ManyModel(object):
def init(self):
self.models=[]

def add_model(self,init_vals):
model = Model(init_vals)
self.model.append(model)

def step(self,time):
for model in self.models:
model.do_step(time)

def get_data_step(self):
data=[]
for model in self.models:
data.append(model.myvalues)
return data



sim=ManyModel()
inits=[] #####list of init_vals
times=[] ####list of times to simulate
for init in intis:
sim.add_model(init)

for time in times:
sim.step(time)
step_data=sim.get_data_step()


So far I have tried to use ray with the decorator @ray.remote over the Model class (1) and the ManyModel class (2) in these two ways:

(1)

############################## (1) ###############
import some_library
import sim_library_with_global_object

@ray.remote
class Model(object):
def __init__(self,init_vals):
#initialize object using some of the global_object from sim_library.
#the Model object have it's own variables not global

def do_step(self,time):
#calculate Model step using the global_object from sim_library
#edit the Model variables with respect to the step


class ManyModel(object):
def init(self):
self.models=[]


def add_model(self,init_vals):
model = Model.remote(init_vals)
self.model.append(model)

def step(self,time):
futures=[]
for model in self.models:
futures.append(model.do_step.remote(time))
return futures

def get_data_step(self,futures):
data=[]
while len(futures)>0:
ready, not_ready = ray.wait(ids)
results=ray.get(ready)
data.append(results)
return data

ray.init()
sim=ManyModel()
inits=[] #####list of init_vals
times=[] ####list of times to simulate
for init in intis:
sim.add_model(init)

for time in times:
sim.step(time)
step_data=sim.get_data_step()


and (2)

########################## (2) #################

import some_library
import sim_library_with_global_object

class Model(object):
def __init__(self,init_vals):
#initialize object using some of the global_object from sim_library.
#the Model object have it's own variables not global

def do_step(self,time):
#calculate Model step using the global_object from sim_library
#edit the Model variables with respect to the step

@ray.remote
class ManyModel(object):
def init(self):
self.models=[]
self.data=[]


def add_model(self,init_vals):
model = Model(init_vals)
self.model.append(model)

def step(self,time):
for model in self.models:
model.do_step(time)

def get_data_step(self):
self.data=[]
for model in self.models:
self.data.append(model.myvalues)
return self.data


ray.init()
sim=ManyModel.remote()
inits=[] #####list of init_vals
times=[] ####list of times to simulate
for init in intis:
sim.add_model.remote(init)

for time in times:
sim.step.remote(time)
future=sim.get_data_step.remote()
step_data=ray.get(future)


In both ways, I'm not getting any benefit from using ray library. Can you help me in the usage?

Updates of Method (1) The problem of the first approach is that I receive this message of warning

2020-11-09 11:33:20,517 WARNING worker.py:1779 -- WARNING: 12 PYTHON workers have been started. This could be a result of using a large number of actors, or it could be a consequence of using nested tasks (see https://github.com/ray-project/ray/issues/3644) for some a discussion of workarounds.

With 10 x Model this are the performance results: With no use of ray: 10 x Model -> do_step 0.11 With ray (1): 10 x Model -> do_step 0.22

Moreover, every time I create an Actor with the method (1) it makes a copy of all the global_objects fo the imported libraries and the ram consumption get crazy. I need to lunch simulation with more than 100k+ Model objects.

In general, I did not understand if in ray it is a good idea to create many actors or not.

Continue reading...