Dask Cluster

Follow below steps to set up Dask Cluster

OS - Ubuntu 16.04

Cloud - AWS

# Install pip

sudo apt-get update

# Install pip

sudo apt-get install python-pip

# Install Docker

    $ curl -fsSL https://get.docker.com -o get-docker.sh
    $ sh get-docker.sh

# Add your user to the docker group.

sudo usermod -aG docker $USER

# install git

# clone https://github.com/dask/dask-docker

$ git clone https://github.com/dask/dask-docker

#go to  cd dask-docker/ directory

# install docker compose

$ sudo curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose

sudo chmod +x /usr/local/bin/docker-compose # executable permission given

# test installation

docker-compose --version

# docker-compose up

# Run shceduler
docker run -it --network host daskdev/dask

# run worker
docker run -it --network host daskdev/dask dask-worker 10.0.0.60:8786

#create notebook image
docker-compose build notebook

#run notbook
docker run -it 39f3e70c6c6a

http://34.219.231.56:8888/?token=aae3eecd23175ad66171e803b1b37eac892907026b77d98c



from dask.distributed import Client
client = Client('10.0.0.60:8786')
client.ncores()



PORT="$(docker ps|grep 4ac7d0c56275|sed 's/.*0.0.0.0://g'|sed 's/->.*//g')"


docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}'  4ac7d0c56275


#run shceduler

 docker run -it --network host daskdev/dask dask-scheduler


# worker on same node

docker run -it --network host daskdev/dask dask-worker localhost:8786

# worker on different node
 docker run -it --network bridge daskdev/dask dask-worker <10.0.0.60 replace with scheduler IP>:8786